From d637a758feba5999d8a8d2c7876651519ba74e0f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 28 Oct 2026 09:26:09 -0600 Subject: [PATCH 01/36] feat: add the Comet Native Executors wire protocol A Spark driver submits a query to a Comet Native Executor cluster as a QueryRequest: a stage graph of Comet native blocks, connected by the exchanges between them, with each stage's Spark output schema and, for stages that read files, each partition's plan. A Fragment carries a stage from the scheduler to its executors. The proto crate also exports a fingerprint of the .proto files, so that a cluster can refuse a query from a client that speaks another plan format. See #7288. --- native/proto/build.rs | 26 ++++++ native/proto/src/lib.rs | 10 +++ native/proto/src/proto/native_executors.proto | 82 +++++++++++++++++++ 3 files changed, 118 insertions(+) create mode 100644 native/proto/src/proto/native_executors.proto diff --git a/native/proto/build.rs b/native/proto/build.rs index 634888e29b7..59c1dd9c166 100644 --- a/native/proto/build.rs +++ b/native/proto/build.rs @@ -34,8 +34,34 @@ fn main() -> Result<()> { "src/proto/partitioning.proto", "src/proto/operator.proto", "src/proto/config.proto", + "src/proto/native_executors.proto", ], &["src/proto"], )?; + + println!( + "cargo:rustc-env=COMET_PROTO_FINGERPRINT={:016x}", + proto_fingerprint("src/proto")? + ); Ok(()) } + +/// A hash of every .proto file, so two builds can tell whether they speak the same plan format. +/// FNV-1a rather than the standard library's hasher, whose output can change between Rust +/// releases: a driver and a native executor cluster may be built with different compilers. +fn proto_fingerprint(dir: &str) -> Result { + let mut paths: Vec<_> = fs::read_dir(dir)? + .map(|entry| entry.map(|e| e.path())) + .collect::>()?; + paths.retain(|path| path.extension().is_some_and(|ext| ext == "proto")); + paths.sort(); + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for path in paths { + let name = path.file_name().unwrap().to_string_lossy().into_owned(); + for byte in name.bytes().chain(fs::read(&path)?) { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0100_0000_01b3); + } + } + Ok(hash) +} diff --git a/native/proto/src/lib.rs b/native/proto/src/lib.rs index b1d6a173bfb..c0ff3200922 100644 --- a/native/proto/src/lib.rs +++ b/native/proto/src/lib.rs @@ -51,6 +51,16 @@ pub mod spark_config { include!(concat!("generated", "/spark.spark_config.rs")); } +// Include generated modules from .proto files. +#[allow(missing_docs)] +pub mod spark_native_executors { + include!(concat!("generated", "/spark.spark_native_executors.rs")); +} + +/// A hash of the .proto files this crate was built from. A Comet Native Executor cluster +/// requires a driver's to match its own. +pub const PROTO_FINGERPRINT: &str = env!("COMET_PROTO_FINGERPRINT"); + #[cfg(test)] mod tests { use super::spark_operator::{ diff --git a/native/proto/src/proto/native_executors.proto b/native/proto/src/proto/native_executors.proto new file mode 100644 index 00000000000..6c0ba50afb4 --- /dev/null +++ b/native/proto/src/proto/native_executors.proto @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + +// Messages a Spark driver exchanges with a Comet Native Executor cluster. See issue #7288. + +syntax = "proto3"; + +package spark.spark_native_executors; + +import "operator.proto"; +import "partitioning.proto"; + +option java_package = "org.apache.comet.serde"; + +// A query submitted by a Spark driver to a Comet Native Executor cluster, as the command of +// an Arrow Flight GetFlightInfo request. +message QueryRequest { + // The Comet version and proto fingerprint of the client's build. The scheduler requires both + // to match its own. + string comet_version = 1; + string proto_fingerprint = 2; + // Spark and Comet configs that native execution reads, as a Spark task would pass them. + map conf = 3; + // Stages in topological order: every stage's producers come before it. The last stage is + // the root, and its output is the query result. + repeated Stage stages = 4; +} + +message Stage { + // The stage's native block. Its input leaves (Scan and ShuffleScan) read the stage's + // inputs, matched in the order a depth-first walk of the block meets them. + spark.spark_operator.Operator plan = 1; + repeated StageInput inputs = 2; + // For a stage that reads files: the block with each partition's files injected, one + // serialized Operator per partition. Empty for a stage whose partitions all run `plan`. + repeated bytes partition_plans = 3; + // The number of partitions of a stage that reads no exchange: the length of partition_plans + // for a stage that reads files, or otherwise the block's own, such as a range's number of + // slices. A stage that reads exchanges has as many partitions as they deliver. + uint32 num_partitions = 4; + // The block's output as Spark declares it. Each partition's output is cast to it before it + // leaves the stage, as a Spark task's shuffle writer does, because a hash exchange places + // rows by the types of their values too. + repeated spark.spark_operator.SparkStructField output_schema = 5; +} + +// How a scheduler sends a stage to its executors, inside the task plans Ballista ships. The +// scheduler and the executors come from one build, so this needs no version check of its own. +message Fragment { + // The block every partition runs. Empty for a stage that reads files, where each partition + // runs its own copy, which the partition descriptor carries. + bytes block = 1; + repeated spark.spark_operator.SparkStructField output_schema = 2; + // Whether every partition reads all of each input, one entry per input. + repeated bool broadcast = 3; + map conf = 4; +} + +message StageInput { + // Index into QueryRequest.stages of the stage producing this input. + uint32 producer = 1; + // How the producer's output is partitioned for this stage. Unset for a broadcast. + spark.spark_partitioning.Partitioning partitioning = 2; + // Every partition of this stage reads the producer's whole output. + bool broadcast = 3; +} From 09ebe654adaa4298f4373fe52e2d87e6e545d470 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 28 Oct 2026 09:26:09 -0600 Subject: [PATCH 02/36] feat: run Comet native blocks as DataFusion operators outside Spark CometFragmentExec runs a stage's native block for each of its partitions, with the block's input leaves fed by other DataFusion plans, such as a distributed engine's shuffle readers, instead of by the JVM. PhysicalPlanner::with_native_inputs is the seam: an input leaf (Scan or ShuffleScan) takes the next native input, cast to the types Spark declared for it, and an expression that calls into the JVM fails to plan. Each partition's output is cast to the stage's declared Spark schema before it leaves the stage, as a Spark task's shuffle writer casts its input, since a hash exchange places rows by the types of their values too. The block runs in a Comet session over the host's memory pool and disk manager. A partition descriptor child, an in-memory source with one row per partition, carries each partition's index in its stage and, for a stage that reads files, the partition's own copy of the block. An engine that splits a stage between tasks slices it with the stage's other leaves. block_schema plans a block with empty inputs, so that a scheduler can check a block before running it. See #7288. --- native/core/src/execution/jni_api.rs | 24 +- native/core/src/execution/mod.rs | 1 + .../execution/native_executors/fragment.rs | 440 ++++++++++++++++++ .../src/execution/native_executors/mod.rs | 33 ++ native/core/src/execution/operators/scan.rs | 3 +- native/core/src/execution/planner.rs | 133 +++++- native/core/src/execution/planner/shuffle.rs | 2 +- native/core/src/execution/spark_config.rs | 3 + 8 files changed, 617 insertions(+), 22 deletions(-) create mode 100644 native/core/src/execution/native_executors/fragment.rs create mode 100644 native/core/src/execution/native_executors/mod.rs diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index 50df7a786b0..d915abfed0b 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -1068,11 +1068,27 @@ fn prepare_datafusion_session_context( spark_plan: &Operator, ) -> CometResult { let paths = local_dirs.into_iter().map(PathBuf::from).collect(); - let mut rt_config = RuntimeEnvBuilder::new() - .with_disk_manager_builder(plan_disk_manager(paths, max_temp_directory_size)) - .with_object_store_registry(Arc::new(CometObjectStoreRegistry::default())); - rt_config = rt_config.with_memory_pool(memory_pool); + let rt_config = comet_runtime_env(memory_pool) + .with_disk_manager_builder(plan_disk_manager(paths, max_temp_directory_size)); + comet_session_context(rt_config, batch_size, task_cpus, spark_config, spark_plan) +} + +/// The runtime of a Comet session that reserves memory from `memory_pool`. It has no disk +/// manager of its own yet. +pub(crate) fn comet_runtime_env(memory_pool: Arc) -> RuntimeEnvBuilder { + RuntimeEnvBuilder::new() + .with_object_store_registry(Arc::new(CometObjectStoreRegistry::default())) + .with_memory_pool(memory_pool) +} +/// Configures a DataFusion session for `spark_plan` over the runtime `rt_config` builds. +pub(crate) fn comet_session_context( + rt_config: RuntimeEnvBuilder, + batch_size: usize, + task_cpus: usize, + spark_config: &HashMap, + spark_plan: &Operator, +) -> CometResult { let mut session_config = SessionConfig::new() .with_target_partitions(task_cpus) // This DataFusion context is within the scope of an executing Spark Task. We want to set diff --git a/native/core/src/execution/mod.rs b/native/core/src/execution/mod.rs index 39ee2627d53..5088bfd3e46 100644 --- a/native/core/src/execution/mod.rs +++ b/native/core/src/execution/mod.rs @@ -24,6 +24,7 @@ pub mod expressions; pub mod jni_api; pub(crate) mod merge_as_partial; pub(crate) mod metrics; +pub mod native_executors; pub mod operators; pub(crate) mod planner; #[cfg(feature = "python-udf")] diff --git a/native/core/src/execution/native_executors/fragment.rs b/native/core/src/execution/native_executors/fragment.rs new file mode 100644 index 00000000000..4e4aec790e0 --- /dev/null +++ b/native/core/src/execution/native_executors/fragment.rs @@ -0,0 +1,440 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; +use std::fmt; +use std::sync::Arc; + +use arrow::array::{Array, AsArray, BinaryArray, RecordBatch, RecordBatchOptions, UInt32Array}; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef, UInt32Type}; +use datafusion::common::tree_node::TreeNodeRecursion; +use datafusion::common::{internal_datafusion_err, DataFusionError, Result}; +use datafusion::datasource::memory::MemorySourceConfig; +use datafusion::execution::disk_manager::DiskManager; +use datafusion::execution::memory_pool::{MemoryPool, UnboundedMemoryPool}; +use datafusion::execution::TaskContext; +use datafusion::physical_expr::{Distribution, EquivalenceProperties, PhysicalExpr}; +use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::streaming::{PartitionStream, StreamingTableExec}; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, + InputDistributionRequirements, Partitioning, PlanProperties, SendableRecordBatchStream, +}; +use datafusion::prelude::SessionContext; +use datafusion_comet_proto::spark_native_executors::Fragment; +use datafusion_comet_proto::spark_operator::{Operator, SparkStructField}; +use futures::stream::BoxStream; +use futures::{StreamExt, TryStreamExt}; + +use crate::execution::jni_api::{comet_runtime_env, comet_session_context}; +use crate::execution::planner::{ + align_shuffle_writer_input, convert_spark_types_to_arrow_schema, PhysicalPlanner, +}; +use crate::execution::serde::deserialize_op; +use crate::execution::spark_config::{ + SparkConfig, COMET_BATCH_SIZE, COMET_STRICT_TIMESTAMP_LABELS, +}; + +const DEFAULT_BATCH_SIZE: usize = 8192; + +fn descriptor_schema() -> SchemaRef { + Arc::new(Schema::new(vec![ + Field::new("partition", DataType::UInt32, false), + Field::new("plan", DataType::Binary, true), + ])) +} + +/// The partition descriptor of a fragment with `num_partitions` partitions. +/// +/// Partition `i` is a single row holding `i`, the partition's index in its stage, and +/// `plans[i]`, the partition's own copy of the block, when `plans` is not empty. A stage that +/// reads files has one, naming the partition's files. A distributed engine that splits a stage's +/// partitions between tasks slices the descriptor along with the stage's other leaves, so a task +/// still learns each partition's index in the stage: Spark seeds `rand`, +/// `monotonically_increasing_id` and similar expressions with it. +pub fn partition_descriptor( + num_partitions: usize, + plans: &[Vec], +) -> Result> { + let schema = descriptor_schema(); + let partitions = (0..num_partitions) + .map(|index| { + let plan = plans.get(index).map(Vec::as_slice); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(UInt32Array::from(vec![index as u32])), + Arc::new(BinaryArray::from(vec![plan])), + ], + )?; + Ok(vec![batch]) + }) + .collect::>>()?; + Ok(MemorySourceConfig::try_new_exec(&partitions, schema, None)?) +} + +/// Runs a Comet native block for each partition of a stage. +/// +/// The first child is the partition descriptor (see [`partition_descriptor`]), and each other +/// child is an input of the block, matched to the block's input leaves in the order Comet's +/// planner meets them. Partition `p` of the fragment reads partition `p` of each partitioned +/// input, and the single partition of each broadcast input. +#[derive(Debug)] +pub struct CometFragmentExec { + fragment: Arc, + /// The partition descriptor, then the inputs. + children: Vec>, + props: Arc, +} + +impl CometFragmentExec { + pub fn try_new(fragment: Arc, children: Vec>) -> Result { + if fragment + .output_schema + .iter() + .any(|field| field.data_type.is_none()) + { + return Err(internal_datafusion_err!( + "CometFragmentExec's output schema has a field without a type" + )); + } + let Some((descriptor, inputs)) = children.split_first() else { + return Err(internal_datafusion_err!( + "CometFragmentExec needs a partition descriptor" + )); + }; + if inputs.len() != fragment.broadcast.len() { + return Err(internal_datafusion_err!( + "CometFragmentExec has {} inputs but {} broadcast flags", + inputs.len(), + fragment.broadcast.len() + )); + } + let partitions = descriptor.output_partitioning().partition_count(); + for (index, (input, broadcast)) in inputs.iter().zip(&fragment.broadcast).enumerate() { + let expected = if *broadcast { 1 } else { partitions }; + let actual = input.output_partitioning().partition_count(); + if actual != expected { + return Err(internal_datafusion_err!( + "CometFragmentExec input {index} has {actual} partitions, expected {expected}" + )); + } + } + let props = Arc::new(PlanProperties::new( + EquivalenceProperties::new(fragment_schema(&fragment.output_schema)), + Partitioning::UnknownPartitioning(partitions), + EmissionType::Incremental, + Boundedness::Bounded, + )); + Ok(Self { + fragment, + children, + props, + }) + } + + pub fn fragment(&self) -> &Arc { + &self.fragment + } +} + +/// The schema of a fragment's output: the types Spark declared for it, every column nullable. +/// The cast to those types keeps a column nullable wherever the block's output is, even where +/// Spark knows it has no nulls, such as a join key a filter has checked, so a fragment can't +/// promise more. Nothing between stages relies on a column having no nulls. +fn fragment_schema(output_schema: &[SparkStructField]) -> SchemaRef { + let declared = convert_spark_types_to_arrow_schema(output_schema); + let fields: Vec = declared + .fields() + .iter() + .map(|field| field.as_ref().clone().with_nullable(true)) + .collect(); + Arc::new(Schema::new(fields)) +} + +impl DisplayAs for CometFragmentExec { + fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "CometFragmentExec: block={} bytes, broadcast={:?}", + self.fragment.block.len(), + self.fragment.broadcast + ) + } +} + +impl ExecutionPlan for CometFragmentExec { + fn name(&self) -> &str { + "CometFragmentExec" + } + + fn properties(&self) -> &Arc { + &self.props + } + + fn children(&self) -> Vec<&Arc> { + self.children.iter().collect() + } + + /// The descriptor's and the partitioned inputs' partitions line up with this plan's, while a + /// broadcast input is read whole by every partition, so a distributed engine must not split + /// it between tasks. + fn input_distribution_requirements(&self) -> InputDistributionRequirements { + let inputs = self.fragment.broadcast.iter().map(|broadcast| { + if *broadcast { + Distribution::SinglePartition + } else { + Distribution::UnspecifiedDistribution + } + }); + InputDistributionRequirements::new( + std::iter::once(Distribution::UnspecifiedDistribution) + .chain(inputs) + .collect(), + ) + } + + /// Repartitioning an input for parallelism would break its alignment with the descriptor. + fn benefits_from_input_partitioning(&self) -> Vec { + vec![false; self.children.len()] + } + + /// The block's expressions are in its serialized plan, so there are none to visit here. + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> Result, + ) -> Result { + Ok(TreeNodeRecursion::Continue) + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + Ok(Arc::new(Self::try_new( + Arc::clone(&self.fragment), + children, + )?)) + } + + fn execute( + &self, + partition: usize, + context: Arc, + ) -> Result { + let descriptor = Arc::clone(&self.children[0]); + let inputs = self.children[1..] + .iter() + .zip(&self.fragment.broadcast) + .map(|(input, broadcast)| { + let input_partition = if *broadcast { 0 } else { partition }; + fragment_input(Arc::clone(input), input_partition, Arc::clone(&context)) + }) + .collect::>>()?; + let fragment = Arc::clone(&self.fragment); + let schema = self.schema(); + let stream = futures::stream::once(async move { + let (index, own_block) = read_descriptor(&descriptor, partition, &context).await?; + // Planning can block on Comet's own runtime, which panics on a runtime thread. + tokio::task::spawn_blocking(move || { + let block = own_block.as_deref().unwrap_or(&fragment.block); + run_block(block, index, inputs, &fragment, schema, &context) + }) + .await + .map_err(|e| DataFusionError::External(Box::new(e)))? + }) + .try_flatten(); + Ok(Box::pin(RecordBatchStreamAdapter::new( + self.schema(), + stream, + ))) + } +} + +/// Reads partition `partition` of a fragment's descriptor: the partition's index in its stage, +/// and the partition's own copy of the block, if it has one. +async fn read_descriptor( + descriptor: &Arc, + partition: usize, + context: &Arc, +) -> Result<(u32, Option>)> { + let batches: Vec = descriptor + .execute(partition, Arc::clone(context))? + .try_collect() + .await?; + let batch = match batches.as_slice() { + [batch] if batch.num_rows() == 1 => batch, + _ => { + return Err(internal_datafusion_err!( + "A fragment's partition descriptor must have one row per partition" + )) + } + }; + let index = batch.column(0).as_primitive::().value(0); + let plans = batch.column(1).as_binary::(); + let plan = (!plans.is_null(0)).then(|| plans.value(0).to_vec()); + Ok((index, plan)) +} + +/// Plans `block` for the partition with index `index` in its stage, feeding its input leaves from +/// `inputs`, and starts it. +/// +/// Its output is cast to the fragment's output schema, the types Spark declared for it, before it +/// leaves the stage, as a Spark task's shuffle writer casts its input: a hash exchange places a +/// row by the types of its key values as well as by the values. A wrong timezone label on a +/// timestamp fails instead in strict test runs, as it does there. Each batch then carries +/// `schema`, the fragment's, which a distributed engine checks it against. +fn run_block( + block: &[u8], + index: u32, + inputs: Vec>, + fragment: &Fragment, + schema: SchemaRef, + context: &TaskContext, +) -> Result>> { + let spark_plan = deserialize_op(block)?; + let session = fragment_session( + &spark_plan, + &fragment.conf, + Arc::clone(context.memory_pool()), + Some(Arc::clone(&context.runtime_env().disk_manager)), + )?; + let planner = + PhysicalPlanner::new(Arc::clone(&session), index as i32).with_native_inputs(inputs); + let root = plan_block(&spark_plan, planner)?; + let root = align_shuffle_writer_input( + root, + &fragment.output_schema, + fragment.conf.get_bool(COMET_STRICT_TIMESTAMP_LABELS), + )?; + let stream = root.execute(0, session.task_ctx())?; + // The session holds state that the block's operators may use while they run. + Ok(stream + .map(move |batch| { + let _session = &session; + let batch = batch?; + // The cast made the names and types the declared ones, so this changes only + // nullability and metadata. + let options = RecordBatchOptions::new().with_row_count(Some(batch.num_rows())); + Ok(RecordBatch::try_new_with_options( + Arc::clone(&schema), + batch.columns().to_vec(), + &options, + )?) + }) + .boxed()) +} + +/// The Comet session a block runs in. It reserves memory from `memory_pool` and spills through +/// `disk_manager`, the host's, so that spill files go where the host keeps them. +fn fragment_session( + spark_plan: &Operator, + conf: &HashMap, + memory_pool: Arc, + disk_manager: Option>, +) -> Result> { + let mut runtime = comet_runtime_env(memory_pool); + runtime.disk_manager = disk_manager; + let session = comet_session_context( + runtime, + conf.get_usize(COMET_BATCH_SIZE, DEFAULT_BATCH_SIZE), + 1, + conf, + spark_plan, + ) + .map_err(|e| DataFusionError::External(Box::new(e)))?; + Ok(Arc::new(session)) +} + +fn plan_block(spark_plan: &Operator, planner: PhysicalPlanner) -> Result> { + let planner = planner.with_sql_text_pool(spark_plan); + let (_, _, root) = planner.create_plan(spark_plan, &mut vec![], 1)?; + Ok(Arc::clone(&root.native_plan)) +} + +/// The output schema of `block`, found by planning it without its data: each input leaf is empty, +/// and each file scan builds no object store for its files, since nothing reads them, so this +/// uses no storage credentials. +/// +/// Fails where running the block would fail at planning, such as for an operator or expression +/// this build doesn't support, or one that calls into the JVM. A problem with the files or their +/// store shows only when the block runs. Planning can block on Comet's own runtime, so don't call +/// this on an async runtime's thread. +pub fn block_schema(block: &[u8], conf: &HashMap) -> Result { + let spark_plan = deserialize_op(block)?; + let session = fragment_session( + &spark_plan, + conf, + Arc::new(UnboundedMemoryPool::default()), + None, + )?; + let planner = PhysicalPlanner::new(session, 0).without_data(); + Ok(plan_block(&spark_plan, planner)?.schema()) +} + +/// One partition of a host plan, read with the host's task context: what a block's input leaf +/// reads. Comet's planner sees it as a single-partition leaf. +fn fragment_input( + input: Arc, + partition: usize, + context: Arc, +) -> Result> { + let schema = input.schema(); + let stream = FragmentInput { + input, + partition, + context, + schema: Arc::clone(&schema), + }; + Ok(Arc::new(StreamingTableExec::try_new( + schema, + vec![Arc::new(stream)], + None, + vec![], + false, + None, + )?)) +} + +#[derive(Debug)] +struct FragmentInput { + input: Arc, + partition: usize, + context: Arc, + schema: SchemaRef, +} + +impl PartitionStream for FragmentInput { + fn schema(&self) -> &SchemaRef { + &self.schema + } + + /// Ignores the block's task context: the input belongs to the host's plan. + fn execute(&self, _ctx: Arc) -> SendableRecordBatchStream { + match self + .input + .execute(self.partition, Arc::clone(&self.context)) + { + Ok(stream) => stream, + Err(e) => Box::pin(RecordBatchStreamAdapter::new( + Arc::clone(&self.schema), + futures::stream::once(async { Err(e) }), + )), + } + } +} diff --git a/native/core/src/execution/native_executors/mod.rs b/native/core/src/execution/native_executors/mod.rs new file mode 100644 index 00000000000..fc0b9d6b2f4 --- /dev/null +++ b/native/core/src/execution/native_executors/mod.rs @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Running Comet plans outside a Spark executor, on Comet Native Executors. See +//! . +//! +//! A Spark driver splits a query into stages at its exchanges. Each stage is one native block: +//! the serialized `Operator` tree a Spark task would run, whose input leaves (`Scan` and +//! `ShuffleScan`) read the stage's inputs. [`CometFragmentExec`] runs such a block for each +//! partition of a stage, reading its inputs from other DataFusion plans, such as a distributed +//! engine's shuffle readers, instead of from the JVM. + +mod fragment; + +pub use fragment::{block_schema, partition_descriptor, CometFragmentExec}; + +/// The version of this Comet build. A Comet Native Executor cluster requires a driver's to match +/// its own, together with [`datafusion_comet_proto::PROTO_FINGERPRINT`]. +pub const COMET_VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/native/core/src/execution/operators/scan.rs b/native/core/src/execution/operators/scan.rs index b3ec14f2e96..64b9ce813f5 100644 --- a/native/core/src/execution/operators/scan.rs +++ b/native/core/src/execution/operators/scan.rs @@ -192,7 +192,8 @@ fn import_column(col: &ArrayRef) -> Result { Ok(decode_string_arrays(col)?) } -fn schema_from_data_types(data_types: &[DataType]) -> SchemaRef { +/// The schema of an input leaf whose columns have `data_types`, without dictionary types. +pub(crate) fn schema_from_data_types(data_types: &[DataType]) -> SchemaRef { let fields = data_types .iter() .enumerate() diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 10cef2334f1..838edc906a7 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -29,6 +29,8 @@ mod shuffle; mod window; mod write; +pub(crate) use shuffle::align_shuffle_writer_input; + // Glue that wires the optional Delta integration into core's plan-tree builder. // Compiled only under `--features contrib-delta`; default builds carry zero Delta // surface. The bulk of the Delta logic lives in the `comet-contrib-delta` crate -- @@ -50,10 +52,11 @@ use crate::execution::operators::TopKReaderFilterExec; #[cfg(feature = "python-udf")] use crate::execution::operators::{ArrowPythonUdfExec, ArrowPythonUdfSpec}; use crate::execution::{ - operators::{ExecutionError, ScanExec, ShuffleScanExec}, + operators::{schema_from_data_types, ExecutionError, ScanExec, ShuffleScanExec}, planner::expression_registry::ExpressionRegistry, planner::operator_registry::OperatorRegistry, serde::{to_arrow_datatype, to_arrow_field}, + shuffle::SchemaAlignExec, }; use crate::jvm_bridge::{jni_call, JVMClasses, OutputStreamSink, ShufflePartitionPusher}; use arrow::compute::CastOptions; @@ -127,7 +130,10 @@ use jni::objects::{Global, JObject}; use log::warn; use num::{BigInt, ToPrimitive}; use object_store::path::Path; -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::{HashMap, VecDeque}, + sync::{Arc, Mutex}, +}; pub type PlanCreationResult = Result<(Vec, Vec, Arc), ExecutionError>; @@ -207,6 +213,18 @@ pub struct PhysicalPlanner { /// Whether a boundary that would cast a timestamp's timezone label to the declared one fails /// instead. Test-only; see `spark.comet.testing.strictTimestampLabels.enabled`. strict_timestamp_labels: bool, + /// What feeds the block's input leaves (`Scan` and `ShuffleScan`) in place of the JVM. Set + /// when Comet runs outside a Spark executor, which also means there is no JVM to call; see + /// [`crate::execution::native_executors`]. + native_inputs: Option>, +} + +/// What feeds a block's input leaves when Comet runs outside a Spark executor. +enum NativeInputs { + /// The plans left to feed the leaves, in the order the planner meets them. + Plans(Mutex>>), + /// Nothing: the block is planned without its data. See [`PhysicalPlanner::without_data`]. + Empty, } impl Default for PhysicalPlanner { @@ -229,6 +247,7 @@ impl PhysicalPlanner { output_stream: None, float_operands: FloatOperands::Normalize, strict_timestamp_labels: false, + native_inputs: None, } } @@ -321,6 +340,92 @@ impl PhysicalPlanner { self } + /// Feeds the block's input leaves from `inputs` instead of the JVM, one per leaf in the order + /// the planner meets them. Each input's output is cast to the types Spark declared for its + /// leaf. Expressions that call into the JVM fail to plan, since there is no JVM. + pub fn with_native_inputs(mut self, inputs: Vec>) -> Self { + self.native_inputs = Some(Arc::new(NativeInputs::Plans(Mutex::new(inputs.into())))); + self + } + + /// Like [`Self::with_native_inputs`], but plans a block without its data, to check it and + /// learn its schema: each input leaf is empty, and each file scan is planned with its files + /// but builds no object store for them, since nothing reads them. Planning then uses no + /// storage credentials. + pub fn without_data(mut self) -> Self { + self.native_inputs = Some(Arc::new(NativeInputs::Empty)); + self + } + + /// Whether the planner plans the block without its data; see [`Self::without_data`]. + fn plans_without_data(&self) -> bool { + matches!(self.native_inputs.as_deref(), Some(NativeInputs::Empty)) + } + + /// Registers the object store a scan reads `file` through, configured by `options`, and + /// returns its URL and backend. When planning without data, the scan gets the local file + /// system instead, which is always registered, and no store is built. + fn scan_store( + &self, + file: &str, + options: &HashMap, + ) -> Result<(ObjectStoreUrl, ObjectStoreBackend), ExecutionError> { + if self.plans_without_data() { + return Ok(( + ObjectStoreUrl::local_filesystem(), + ObjectStoreBackend::Local, + )); + } + let (url, _, backend) = prepare_object_store_with_configs( + self.session_ctx.runtime_env(), + file.to_string(), + options, + )?; + Ok((url, backend)) + } + + /// Plans `spark_plan` as the next native input when it is an input leaf and the planner has + /// native inputs. `None` otherwise. + fn native_input_leaf( + &self, + spark_plan: &Operator, + ) -> Result>, ExecutionError> { + let Some(native_inputs) = &self.native_inputs else { + return Ok(None); + }; + let fields = match spark_plan.op_struct.as_ref() { + Some(OpStruct::Scan(scan)) => &scan.fields, + Some(OpStruct::ShuffleScan(scan)) => &scan.fields, + _ => return Ok(None), + }; + let data_types = fields.iter().map(to_arrow_datatype).collect_vec(); + let schema = schema_from_data_types(&data_types); + let input = match native_inputs.as_ref() { + NativeInputs::Plans(plans) => { + let input = plans.lock().unwrap().pop_front().ok_or_else(|| { + GeneralError("The plan has more input leaves than native inputs".to_string()) + })?; + SchemaAlignExec::try_new_or_passthrough(input, &schema)? + } + NativeInputs::Empty => Arc::new(EmptyExec::new(schema)), + }; + Ok(Some(Arc::new(SparkPlan::new( + spark_plan.plan_id, + input, + vec![], + )))) + } + + /// Fails to plan `what`, which calls into the JVM, when the plan runs without one. + fn require_jvm(&self, what: &str) -> Result<(), ExecutionError> { + if self.native_inputs.is_some() { + return Err(GeneralError(format!( + "{what} calls into the JVM, so it cannot run on Comet Native Executors" + ))); + } + Ok(()) + } + pub(crate) fn strict_timestamp_labels(&self) -> bool { self.strict_timestamp_labels } @@ -507,7 +612,7 @@ impl PhysicalPlanner { partition_files: &SparkFilePartition, ) -> Result<(ObjectStoreUrl, ObjectStoreBackend, Vec), ExecutionError> { let one_file = match partition_files.partitioned_file.first() { - Some(f) => f.file_path.clone(), + Some(f) => f.file_path.as_str(), None => { // Empty partition: no store to resolve; the URL and backend are unused because // the file group is empty and build_parquet_scan_plan returns EmptyExec. @@ -523,11 +628,8 @@ impl PhysicalPlanner { .iter() .map(|(k, v)| (k.clone(), v.clone())) .collect(); - let (object_store_url, _, object_store_backend) = prepare_object_store_with_configs( - self.session_ctx.runtime_env(), - one_file, - &object_store_options, - )?; + let (object_store_url, object_store_backend) = + self.scan_store(one_file, &object_store_options)?; let files = self.get_partitioned_files(partition_files, &object_store_options)?; Ok((object_store_url, object_store_backend, files)) } @@ -857,6 +959,7 @@ impl PhysicalPlanner { Ok(Arc::new(CastExpr::new(child, data_type, None))) } ExprStruct::Subquery(expr) => { + self.require_jvm("A scalar subquery")?; let id = expr.id; let data_type = to_arrow_datatype(expr.datatype.as_ref().unwrap()); Ok(Arc::new(Subquery::new(self.exec_context_id, id, data_type))) @@ -1023,6 +1126,7 @@ impl PhysicalPlanner { ))) } ExprStruct::JvmScalarUdf(udf) => { + self.require_jvm(&format!("The JVM UDF {}", udf.class_name))?; let args = udf .args .iter() @@ -1524,6 +1628,10 @@ impl PhysicalPlanner { inputs: &mut Vec>>>, partition_count: usize, ) -> PlanCreationResult { + if let Some(input) = self.native_input_leaf(spark_plan)? { + return Ok((vec![], vec![], input)); + } + // Try to use the modular registry first - this automatically handles any registered operator types if OperatorRegistry::global().can_handle(spark_plan) { return OperatorRegistry::global().create_plan( @@ -1776,14 +1884,7 @@ impl PhysicalPlanner { // Build the store from this partition's files; other partitions may read // from a different one. let object_store_url = match partition.partitioned_file.first() { - Some(file) => { - prepare_object_store_with_configs( - self.session_ctx.runtime_env(), - file.file_path.clone(), - &object_store_options, - )? - .0 - } + Some(file) => self.scan_store(&file.file_path, &object_store_options)?.0, None => ObjectStoreUrl::local_filesystem(), }; let files = self.get_partitioned_files(partition, &object_store_options)?; diff --git a/native/core/src/execution/planner/shuffle.rs b/native/core/src/execution/planner/shuffle.rs index f5fb1f869ac..9d3b073879d 100644 --- a/native/core/src/execution/planner/shuffle.rs +++ b/native/core/src/execution/planner/shuffle.rs @@ -292,7 +292,7 @@ impl OperatorBuilder for ShuffleScanBuilder { /// /// With `strict_timestamp_labels`, a timestamp labelled with another timezone than declared fails /// planning instead of being cast. The child is native, so the label is wrong inside its plan too. -fn align_shuffle_writer_input( +pub(crate) fn align_shuffle_writer_input( child: Arc, expected_proto: &[spark_operator::SparkStructField], strict_timestamp_labels: bool, diff --git a/native/core/src/execution/spark_config.rs b/native/core/src/execution/spark_config.rs index 511b5733388..302719f4a98 100644 --- a/native/core/src/execution/spark_config.rs +++ b/native/core/src/execution/spark_config.rs @@ -32,6 +32,9 @@ pub(crate) const COMET_EXEC_SPILL_COMPRESSION_CODEC: &str = "spark.comet.exec.spill.compression.codec"; pub(crate) const COMET_NATIVE_UDF_ENABLED: &str = "spark.comet.nativeUdf.enabled"; pub(crate) const COMET_NATIVE_UDF_ALLOWED_PATHS: &str = "spark.comet.nativeUdf.allowedPaths"; +/// A Spark task passes the batch size to native code apart from the other configs, so only a host +/// other than a Spark executor reads it from them. +pub(crate) const COMET_BATCH_SIZE: &str = "spark.comet.batchSize"; pub(crate) const COMET_STRICT_TIMESTAMP_LABELS: &str = "spark.comet.testing.strictTimestampLabels.enabled"; pub(crate) const SPARK_EXECUTOR_CORES: &str = "spark.executor.cores"; From 8a4abe1878d4ebeb70df341d730b0b0746bd3a0c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 28 Oct 2026 09:26:09 -0600 Subject: [PATCH 03/36] feat: add comet-scheduler and comet-executor built on DataFusion Ballista A new Cargo workspace, native/native-executors, builds the two processes of a Comet Native Executor cluster on the Ballista 55 crates. It is a workspace of its own, so the main Comet build never depends on Ballista. comet-scheduler wraps Ballista's scheduler in an Arrow Flight service. GetFlightInfo takes a QueryRequest, checks that the client runs the same Comet build, plans each block with Comet's planner, turns the stage graph into a Ballista physical plan, runs it, and returns one endpoint per result partition with an opaque ticket for the executor's DoGet. Hash exchanges become RepartitionExec(Hash), and single-partition and broadcast exchanges become CoalescePartitionsExec, which Ballista's planner turns into stage boundaries. A stage that reads exchanges has as many partitions as they deliver. Ballista can't plan a range shuffle for a physical plan yet (apache/datafusion-ballista#2680), so a range exchange is read as one partition, and the stage that sorts it runs in a single task. comet-executor is Ballista's executor with Comet's codec, which carries each CometFragmentExec's Fragment through Ballista's stage and task plans. It needs no JVM. Both binaries take their settings from the environment, with Ballista's defaults otherwise. See #7288. --- dev/release/rat_exclude_files.txt | 1 + .../contributor-guide/release_process.md | 4 +- native/Cargo.toml | 3 +- native/native-executors/Cargo.lock | 7468 +++++++++++++++++ native/native-executors/Cargo.toml | 61 + native/native-executors/README.md | 87 + native/native-executors/src/assembly.rs | 190 + .../src/bin/comet-executor.rs | 67 + .../src/bin/comet-scheduler.rs | 81 + native/native-executors/src/codec.rs | 75 + native/native-executors/src/frontend.rs | 259 + native/native-executors/src/lib.rs | 53 + native/native-executors/tests/assembly.rs | 253 + native/native-executors/tests/common/mod.rs | 164 + 14 files changed, 8764 insertions(+), 2 deletions(-) create mode 100644 native/native-executors/Cargo.lock create mode 100644 native/native-executors/Cargo.toml create mode 100644 native/native-executors/README.md create mode 100644 native/native-executors/src/assembly.rs create mode 100644 native/native-executors/src/bin/comet-executor.rs create mode 100644 native/native-executors/src/bin/comet-scheduler.rs create mode 100644 native/native-executors/src/codec.rs create mode 100644 native/native-executors/src/frontend.rs create mode 100644 native/native-executors/src/lib.rs create mode 100644 native/native-executors/tests/assembly.rs create mode 100644 native/native-executors/tests/common/mod.rs diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 3347c2a8ac8..2fd9a988405 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -6,6 +6,7 @@ .claude/skills CLAUDE.md native/Cargo.lock +native/native-executors/Cargo.lock dev/copyright/scala-header.txt dev/diffs/iceberg/*.diff dev/release/requirements.txt diff --git a/docs/source/contributor-guide/release_process.md b/docs/source/contributor-guide/release_process.md index ef31cf5bfc7..0dac5641c8a 100644 --- a/docs/source/contributor-guide/release_process.md +++ b/docs/source/contributor-guide/release_process.md @@ -203,7 +203,9 @@ Create a PR against the main branch to prepare for developing the next release: - Update the Rust crate version to `0.14.0` in `native/Cargo.toml` and in each `contrib/*/native/Cargo.toml`. The contrib crates sit outside the `native/` workspace, so they do not inherit its version. Then run - `cargo update --workspace` in `native/` and in each contrib crate that has its own `Cargo.lock`. + `cargo update --workspace` in `native/`, in `native/native-executors/` (a workspace with its own `Cargo.lock`, + which records the versions of the `native/` crates it uses) and in each contrib crate that has its own + `Cargo.lock`. - Update the Maven version to `0.14.0-SNAPSHOT` in the same set of files listed above (the `pom.xml` files, the Spark test diffs under `dev/diffs`, and the Iceberg test diffs under `dev/diffs/iceberg`). diff --git a/native/Cargo.toml b/native/Cargo.toml index 6f3b771e8ee..1103bed842a 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -20,7 +20,8 @@ default-members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuff members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle", "operators", "comet-udf-sdk", "comet-test-udfs"] # Crates under ../contrib are intentionally NOT workspace members. Core pulls them in as # optional path dependencies when their corresponding contrib feature is enabled. -exclude = ["../contrib"] +# native-executors is its own workspace, so that this one never depends on Ballista. +exclude = ["../contrib", "native-executors"] resolver = "2" [workspace.package] diff --git a/native/native-executors/Cargo.lock b/native/native-executors/Cargo.lock new file mode 100644 index 00000000000..4d21ac8a3e5 --- /dev/null +++ b/native/native-executors/Cargo.lock @@ -0,0 +1,7468 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.7", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "apache-avro" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312c1ea69e5fe9966e0029fb95aca8790100b85aff4f0d3b00a9337c74069a9c" +dependencies = [ + "bigdecimal", + "bon", + "crc32fast", + "digest 0.11.3", + "log", + "miniz_oxide", + "num-bigint 0.4.8", + "ouroboros", + "quad-rand", + "rand 0.10.3", + "regex-lite", + "serde", + "serde_bytes", + "serde_json", + "snap", + "strum 0.28.0", + "thiserror", + "uuid", + "zstd 0.13.3", +] + +[[package]] +name = "ar_archive_writer" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" +dependencies = [ + "object", +] + +[[package]] +name = "arc-swap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380962b3d93b99642e50b9febf72c071e97e8b32d86eae9cf4ef582bb960ae6e" +dependencies = [ + "rustversion", +] + +[[package]] +name = "array-init" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "arrow" +version = "59.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c25317ebddb8f2e6f006759336c95e5ded847a446f3524c27853bfe7d77a69b" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157943c720919e8f5b6731407ce5cd4c9a5ef47a75bf10edaf24b7081b5763ef" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b5b1682924f11ecf6929689c16cbc69efbb6dd141b06ad98ee1cb10d858c43" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.17.1", + "libc", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5d687afc6c927965a8a759f3b68de91933d9dfa2e1bb6e156bac84b85665e92" +dependencies = [ + "bytes", + "half", + "num-bigint 0.5.1", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f23c3379feede42135ef27ca60e67f6e643cd4343ae74e4c8b16c83f55afd85" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64 0.23.1", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "59.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace83d5935421b950fea3a0a3f9972c0837c297ef98ade57127a5661b681112c" +dependencies = [ + "arrow-array", + "arrow-cast", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "regex", +] + +[[package]] +name = "arrow-data" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0a031c7ff5e2bdb443117054fb466af1d4df5b7f0ecbe197713dc8b113b1ba" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-flight" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f998a8cc9ffdc37731b66c0414dccf4f4f11a03d9136d383ef10f95181cd7d" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", + "base64 0.23.1", + "bytes", + "futures", + "once_cell", + "prost", + "prost-types", + "tonic", + "tonic-prost", +] + +[[package]] +name = "arrow-ipc" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5dd1afadda2eb821aaf851bb4c0adef60c30cc0f1a29bd02fb99dd07fb9f58d" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", + "lz4_flex", + "zstd 0.13.3", +] + +[[package]] +name = "arrow-json" +version = "59.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "727b21ebf8894dcd9504d6ab66bffdac338391e1f85cde594526a0210299ea40" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "chrono", + "half", + "indexmap 2.14.2", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + +[[package]] +name = "arrow-ord" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "561b8209faa2cd489764b7fd07d7cb9e8d274d78048c38de59f77c41140cb834" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d8c68b87a569f92b90c46cb0a09d86bd9609ddc9b1bf1fc5a503c8c6907c14e" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e87acb0c6bf6237fca7a50d7b51e801c98a18a5fe6f3bb34f7d77b85ca165388" +dependencies = [ + "bitflags 2.13.2", + "serde_core", + "serde_json", +] + +[[package]] +name = "arrow-select" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c844141958d805b47d1208e0d12aac53fcf388ce8011639fe82aa492adde0bb4" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc130892b1a9552da77971924b390e06cf375d1f7e4344a7b7b40ca4426fa470" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "as-any" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063" + +[[package]] +name = "async-compression" +version = "0.4.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5755640120b24961f8231e05892e4a4f7e0d04e8aeca161508c9c397946970" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "asyncband" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0" +dependencies = [ + "hashbrown 0.17.1", + "slab", +] + +[[package]] +name = "asyncband" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5161fb43cb8f231f06983cb2b8f9ee9cee14be0b1356d8383398cc0f312800d1" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-config" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "939e256cfe0ed68e96d32f194339d88210d5c10685cff0787400158f852c6819" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http", + "sha1 0.10.7", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "576e8da35751dc85409ec1cd15999f692c277a1fbe9336780fe6f15e49dd637d" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-lc-rs" +version = "1.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1622d8446a2d4b2ce0c7eefc73dd43a99779028d5ee5c2dd8073a658ba8a2bc" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "aws-runtime" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a0cc2144962cc5e358467b16e3d82981b1d424a7c3d34602be0036103814a1a" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "bytes-utils", + "fastrand", + "http", + "http-body", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.117.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5316813498fa9c403877007934ae448aa44b18136f58051f8fa3c5b79b44bdc6" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.119.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31db35e5662787b786daba15f431e4393c7ea512b70fa112bd048d1f3ecf877a" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.122.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a10491638d9e560db90c484dbc77d73b8ad202780cdc9b9e92180603acb07ada" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d3d48355cb86b15ccbb38b8bc6b87500c824ee4ed8cbc10a961b435242b5e16" +dependencies = [ + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac 0.13.0", + "http", + "percent-encoding", + "sha2 0.11.1", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de681a9df7ff79823b759ee17061076ec049cd61a8fa28ece4293d89c9f6964e" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-http" +version = "0.64.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba8960956e0ee0ca3f3161e62913a61bc65a7ef638ca3ef1d4c7159f8443e398" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeca69f65362f7375891008ea525fd440ad6db1836458ac188c87b126804114e" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2", + "http", + "hyper", + "hyper-rustls", + "hyper-util", + "pin-project-lite", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.65.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f6835fb38bc6fc8c91e584b0366eb4ff595e5c485134dbf4f0bdeaa18856666" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-query" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165f6265342a313ad6f7a1ac17a0cdba30c522039d3f41d8d6871ce8a2b34ecd" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cef0c41e8ca40f74f68ed8ee4d1ddbde12bb847b95ef9b3c025681ddfae58a" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "bytes", + "fastrand", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06788004dea79018413f381891da6dcca23196acd79c4d51b767fbdacc76eae4" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api-macros", + "aws-smithy-types", + "bytes", + "http", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "aws-smithy-schema" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec8dae1015a6d959a5faf86fbdf77486cfe19e4ccb6ee009d8a46efc45028a4" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http", +] + +[[package]] +name = "aws-smithy-types" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55752025c39c0e094962fd73f1c57014b191f678612e517f219468e575af9822" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "http", + "http-body", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "ryu", + "serde", + "time", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6581618a2fd006159016d1c36881d94d61b4a017f4a8d631310875e18252413" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba0d970d3947bb064d2ae1d823efa186c1dbde88f88afa8f6549946e5d8deee" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "rustc_version", + "tracing", +] + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + +[[package]] +name = "ballista-core" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a090511fa9b3661dbf17444ecbbf5efcd11793e140ddd7fe2fb85f2de88f561" +dependencies = [ + "arrow-flight", + "async-trait", + "chrono", + "datafusion", + "datafusion-proto", + "datafusion-proto-common", + "futures", + "log", + "parking_lot", + "pastey", + "prost", + "rand 0.10.3", + "rustc_version", + "serde", + "tokio", + "tokio-stream", + "tonic", + "tonic-build", + "tonic-prost", + "tonic-prost-build", + "url", + "uuid", +] + +[[package]] +name = "ballista-executor" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403cdac6a4e822e6025348f4f7e43765ba0ca176ca1f225888a9eeb90b9c66ce" +dependencies = [ + "arrow", + "arrow-flight", + "async-trait", + "ballista-core", + "bytesize", + "dashmap", + "datafusion", + "datafusion-proto", + "futures", + "libc", + "log", + "lru", + "memory-stats", + "parking_lot", + "serde", + "sysinfo", + "tempfile", + "tokio", + "tokio-stream", + "tokio-util", + "tonic", + "uuid", +] + +[[package]] +name = "ballista-scheduler" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "845bb1c2c763ae2b50ecfd9debdfe8e092d13a4fefe84c20e9262bb191a1f1e7" +dependencies = [ + "arrow-flight", + "async-trait", + "axum", + "ballista-core", + "dashmap", + "datafusion", + "datafusion-proto", + "ferroid", + "futures", + "http", + "itertools 0.15.0", + "log", + "object_store", + "parking_lot", + "prost", + "rand 0.10.3", + "serde", + "tokio", + "tokio-stream", + "tonic", + "tower-http 0.7.1", + "url-escape", + "uuid", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bigdecimal" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fa3f3d8cbf4dffcfe4991de61d012bef509a409ecbe9dd41049bfe32b4d4653" +dependencies = [ + "autocfg", + "libm", + "num-bigint 0.4.8", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "bimap" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "blake3" +version = "1.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +dependencies = [ + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.1", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bnum" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f781dba93de3a5ef6dc5b17c9958b208f6f3f021623b360fb605ea51ce443f10" +dependencies = [ + "serde", + "serde-big-array", +] + +[[package]] +name = "bon" +version = "3.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a05442cd7005532d0cfedec676b5b9333d2b4c7c3afabe2b1a9ce2742fbe2ee6" +dependencies = [ + "bon-macros", +] + +[[package]] +name = "bon-macros" +version = "3.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93833e97ac7c46ba97045afcf247238c27057538fcf2f06344662ddd5f567be6" +dependencies = [ + "darling 0.24.1", + "ident_case", + "prettyplease 0.3.0", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + +[[package]] +name = "bytesize" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7354288c522e7e980fafd2075d63d1285794c3a6a16cdd492f189ea406e5f18b" + +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + +[[package]] +name = "cc" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "combine" +version = "4.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4356eba6976435498d05a311ee9fc486da3bc81c200d3f0d7dd9703b41913f92" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "comet-native-executors" +version = "0.0.0" +dependencies = [ + "arrow-flight", + "ballista-core", + "ballista-executor", + "ballista-scheduler", + "datafusion", + "datafusion-comet", + "datafusion-comet-proto", + "datafusion-proto", + "env_logger", + "futures", + "log", + "prost", + "prost-types", + "tokio", + "tonic", +] + +[[package]] +name = "comet-udf-sdk" +version = "1.3.0" +dependencies = [ + "arrow", +] + +[[package]] +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +dependencies = [ + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "compression-codecs" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cae36683101b2899f48bd31f8db9839437356188e3b1ff23096e73dab09ba8fe" +dependencies = [ + "bzip2", + "compression-core", + "flate2", + "liblzma", + "memchr", + "zstd 0.14.3", + "zstd-safe 8.2.0", +] + +[[package]] +name = "compression-core" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ebdda89edc534bf8d5dd2857562633b6495557e5846552ade994b99ed13c36a" + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin 0.10.1", +] + +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + +[[package]] +name = "crc32fast" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "ctor" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d" +dependencies = [ + "link-section", + "linktime-proc-macro", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + +[[package]] +name = "ctutils" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d28fb6aa7dd680b3ffd1a3e64e115c177bdba63c0ae7f9cd48129ff05c4fa0" +dependencies = [ + "cmov", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core 0.24.1", + "darling_macro 0.24.1", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.9", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core 0.24.1", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "datafusion" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd5317db77ba6bca25c761a8091b9d294a9f4188e4f9dcd0243e6a57de862ac" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "bzip2", + "chrono", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-datasource-arrow", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", + "datafusion-functions-table", + "datafusion-functions-window", + "datafusion-optimizer", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-session", + "datafusion-sql", + "flate2", + "futures", + "indexmap 2.14.2", + "itertools 0.15.0", + "liblzma", + "log", + "object_store", + "parking_lot", + "parquet", + "sqlparser", + "tempfile", + "tokio", + "url", + "uuid", + "zstd 0.13.3", +] + +[[package]] +name = "datafusion-catalog" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f35d36794509b7bfcbf90e11a1b5e4d047123c34610105e2e80a2048351cdb2f" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools 0.15.0", + "log", + "object_store", + "parking_lot", + "tokio", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6454bfb7f43b83942e2ca2576fd09342eedbd0d323e1eef72c3c8e4d57d7203" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "futures", + "itertools 0.15.0", + "log", + "object_store", + "percent-encoding", +] + +[[package]] +name = "datafusion-comet" +version = "1.3.0" +dependencies = [ + "arrow", + "async-trait", + "aws-config", + "aws-credential-types", + "aws-sdk-sts", + "aws-smithy-runtime-api", + "base64 0.23.1", + "bytes", + "chrono-tz", + "comet-udf-sdk", + "datafusion", + "datafusion-comet-common", + "datafusion-comet-jni-bridge", + "datafusion-comet-operators", + "datafusion-comet-proto", + "datafusion-comet-shuffle", + "datafusion-comet-spark-expr", + "datafusion-datasource", + "datafusion-physical-expr-adapter", + "datafusion-spark", + "futures", + "hdrs", + "iceberg", + "iceberg-storage-opendal", + "itertools 0.15.0", + "jni", + "libloading", + "log", + "log4rs", + "num", + "object_store", + "once_cell", + "opendal", + "parking_lot", + "parquet", + "paste", + "procfs", + "prost", + "rand 0.10.3", + "reqsign-core", + "reqwest 0.12.28", + "serde", + "serde_json", + "tempfile", + "tokio", + "typetag", + "url", + "uuid", +] + +[[package]] +name = "datafusion-comet-common" +version = "1.3.0" +dependencies = [ + "arrow", + "datafusion", + "serde", + "serde_json", + "simdutf8", + "thiserror", +] + +[[package]] +name = "datafusion-comet-jni-bridge" +version = "1.3.0" +dependencies = [ + "arrow", + "datafusion", + "datafusion-comet-common", + "jni", + "lazy_static", + "once_cell", + "parquet", + "paste", + "prost", + "regex", + "serde_json", + "thiserror", +] + +[[package]] +name = "datafusion-comet-operators" +version = "1.3.0" +dependencies = [ + "arrow", + "datafusion", + "datafusion-comet-common", + "datafusion-comet-spark-expr", + "futures", + "parking_lot", +] + +[[package]] +name = "datafusion-comet-proto" +version = "1.3.0" +dependencies = [ + "prost", + "prost-build", +] + +[[package]] +name = "datafusion-comet-shuffle" +version = "1.3.0" +dependencies = [ + "arrow", + "arrow-data", + "arrow-select", + "async-trait", + "crc32c", + "crc32fast", + "datafusion", + "datafusion-comet-common", + "datafusion-comet-jni-bridge", + "datafusion-comet-spark-expr", + "futures", + "itertools 0.15.0", + "jni", + "log", + "lz4_flex", + "simd-adler32", + "snap", + "tokio", + "zstd 0.13.3", +] + +[[package]] +name = "datafusion-comet-spark-expr" +version = "1.3.0" +dependencies = [ + "arrow", + "base64 0.23.1", + "chrono", + "datafusion", + "datafusion-comet-common", + "datafusion-comet-jni-bridge", + "datafusion-functions", + "datafusion-spark", + "datasketches", + "futures", + "jni", + "memchr", + "num", + "paste", + "rand 0.10.3", + "regex", + "ryu", + "serde", + "serde_json", + "smallvec", + "twox-hash", + "uuid", +] + +[[package]] +name = "datafusion-common" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26626e12b224b83351239465dcd00f72d47b02fb5e5b65a643e2dc8ed37f687" +dependencies = [ + "arrow", + "arrow-ipc", + "arrow-schema", + "chrono", + "foldhash 0.2.0", + "half", + "hashbrown 0.17.1", + "hex", + "indexmap 2.14.2", + "itertools 0.15.0", + "libc", + "log", + "num-traits", + "object_store", + "parquet", + "recursive", + "sqlparser", + "tokio", + "uuid", + "web-time", +] + +[[package]] +name = "datafusion-common-runtime" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a106420cbfa980f242c22243ae9a6cce661d4d86ce96cff87dacabf7754a4470" +dependencies = [ + "futures", + "log", + "tokio", +] + +[[package]] +name = "datafusion-datasource" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "210aa1e9525e9454f2399dbb8b0b95510145bd8c1fdc35de74e9ccbf81bcb699" +dependencies = [ + "arrow", + "async-compression", + "async-trait", + "bytes", + "bzip2", + "chrono", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-models", + "datafusion-session", + "flate2", + "futures", + "glob", + "itertools 0.15.0", + "liblzma", + "log", + "object_store", + "parking_lot", + "rand 0.9.5", + "tokio", + "tokio-util", + "url", + "zstd 0.13.3", +] + +[[package]] +name = "datafusion-datasource-arrow" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65b1b041688cad707fa103c3e7f4c6d89b3134ebf67da0129070dad3381851eb" +dependencies = [ + "arrow", + "arrow-ipc", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-models", + "datafusion-session", + "futures", + "itertools 0.15.0", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a7b34e253c4fa5251f174977af910a2f2ec831ec39dbbb56b29ce505cff278b" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-models", + "datafusion-session", + "futures", + "object_store", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-json" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1663338f43be6a5746473008e2d0fbde282577ddf0bae90d6c7386e988d71b29" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-models", + "datafusion-session", + "futures", + "object_store", + "tokio", + "tokio-stream", +] + +[[package]] +name = "datafusion-datasource-parquet" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c521ae91b6d899d7190812881f45fe9463c222c11d90149f5332a2e56555904" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-models", + "datafusion-pruning", + "datafusion-session", + "futures", + "itertools 0.15.0", + "log", + "object_store", + "parking_lot", + "parquet", + "tokio", +] + +[[package]] +name = "datafusion-doc" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2569c2e74a34fcae3a369ab3912d61e41a062592510f11e31b1dae2725ecdc" + +[[package]] +name = "datafusion-execution" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cfdd5ac0769d1e74a885450e49fde8f63ca7ab41eb479a068564aaea80b837" +dependencies = [ + "arrow", + "arrow-buffer", + "async-trait", + "bytes", + "dashmap", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr-common", + "futures", + "log", + "object_store", + "parking_lot", + "parquet", + "pin-project-lite", + "rand 0.9.5", + "tempfile", + "tokio", + "tokio-util", + "url", +] + +[[package]] +name = "datafusion-expr" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86560fd7df4dda924f5474b588c56e7ae8e96dcf158c7327c9cfe9daff448ec5" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-doc", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", + "datafusion-proto-common", + "datafusion-proto-models", + "indexmap 2.14.2", + "itertools 0.15.0", + "recursive", + "serde", + "serde_json", + "sqlparser", +] + +[[package]] +name = "datafusion-expr-common" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e82e313cf655cf6bc2dce6cbd23bcb0d20236a54a1b09c436615e4bac4d8c49f" +dependencies = [ + "arrow", + "datafusion-common", + "indexmap 2.14.2", + "itertools 0.15.0", +] + +[[package]] +name = "datafusion-functions" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "939ae2771da8a3fdc23ab4de610ae8fef9515476626b49d96d0d5c782b105771" +dependencies = [ + "arrow", + "arrow-buffer", + "base64 0.23.1", + "blake2", + "blake3", + "chrono", + "chrono-tz", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", + "datafusion-physical-expr-common", + "hex", + "itertools 0.15.0", + "log", + "md-5 0.11.0", + "memchr", + "num-traits", + "rand 0.9.5", + "regex", + "sha2 0.11.1", + "uuid", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6c3aee9f2bc65716787d651e023743a90b4efbc57a2f679f036632e16fb110" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "half", + "hashbrown 0.17.1", + "log", + "num-traits", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "988b5d3dfb4939d81c2f9fab267063610fe52a843620f2fa6cb2a1598e43369c" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-functions-nested" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08149d433a97c9ea3cdf80de06480ab5c7f5453485f1e452d5b3583c81d5c0dd" +dependencies = [ + "arrow", + "arrow-ord", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr-common", + "hashbrown 0.17.1", + "itertools 0.15.0", + "itoa", + "log", + "memchr", +] + +[[package]] +name = "datafusion-functions-table" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c767567cce2782d4d0bdfa55fab010aaba0bdbe533509e9e0b9a6a3ee29ed59" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "parking_lot", +] + +[[package]] +name = "datafusion-functions-window" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a311c1f558d1f792f1f631c3670da12ee72a20fbf5d6ca151325d66191c03c1" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "log", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fffd54b8acf6ecef284cc0677e01e1b8c71be64fec50612ba00a0ff0d9b41a52" +dependencies = [ + "datafusion-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-macros" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14d6df9763c8fb968b2417d797aef4c55ac2e4a895aaef38c72c16a287aa4a40" +dependencies = [ + "datafusion-doc", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "datafusion-optimizer" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acfb8cea6e05e2e75bae183340cf7d53cccb3db0449e2f1fbc07654d980fd19f" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "indexmap 2.14.2", + "itertools 0.15.0", + "log", + "recursive", + "regex", + "regex-syntax", +] + +[[package]] +name = "datafusion-physical-expr" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f002cef849e10c0597a217ec7891c2796c0f8b42d739f5d24abc24496373406" +dependencies = [ + "arrow", + "arrow-schema", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", + "datafusion-proto-models", + "half", + "hashbrown 0.17.1", + "indexmap 2.14.2", + "itertools 0.15.0", + "parking_lot", + "petgraph", + "recursive", + "tokio", +] + +[[package]] +name = "datafusion-physical-expr-adapter" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82761522c4536e7059d3b185cd99f10525c2bf48f1bd26a0a57e73e19eadc0fb" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "itertools 0.15.0", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8c5c67aa252886f4d6b331c8114797f5b140464e774c4da36889269a1bf1bd8" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr-common", + "datafusion-proto-models", + "hashbrown 0.17.1", + "indexmap 2.14.2", + "itertools 0.15.0", + "parking_lot", + "pin-project", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a49dd3a76e4879012682c59dc3735282e60640c97483546a4113564a60bf8f" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-pruning", + "datafusion-session", + "itertools 0.15.0", + "recursive", +] + +[[package]] +name = "datafusion-physical-plan" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1451d6817e5ac0dd21819b19949d1c420ab51d748f55dd9a96ecb5418a24c97" +dependencies = [ + "arrow", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-schema", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-proto-common", + "datafusion-proto-models", + "futures", + "half", + "hashbrown 0.17.1", + "indexmap 2.14.2", + "itertools 0.15.0", + "log", + "num-traits", + "parking_lot", + "pin-project-lite", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "datafusion-proto" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "481881a3422072554c8c26a177a4f0a67f4d8c0e2a394a79e551e6bd9a5cec4b" +dependencies = [ + "arrow", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-datasource", + "datafusion-datasource-arrow", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-table", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-common", + "datafusion-proto-models", + "object_store", + "prost", +] + +[[package]] +name = "datafusion-proto-common" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a95763d5fa64a88b658ba4ec7314d9c538e2aea749dc88d5ce2afaafbb4c69b8" +dependencies = [ + "arrow", + "datafusion-common", + "prost", +] + +[[package]] +name = "datafusion-proto-models" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c4982d05dee1e0b90b7f3b39845dca7eee5635c550db09158d385b8181bdc6a" +dependencies = [ + "datafusion-common", + "datafusion-proto-common", + "prost", +] + +[[package]] +name = "datafusion-pruning" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f11f0dbe2b5a642f6106e44de8bbd6654fa7220ca14f32313d4efd5514a86968" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-datasource", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "log", +] + +[[package]] +name = "datafusion-session" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62dc5ce37a5ce1ff5362949776829aded2529a11dbaa13d24b0ffa6a059e106a" +dependencies = [ + "arrow-schema", + "async-trait", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", +] + +[[package]] +name = "datafusion-spark" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c3d6f76463984c4ed8587c52ed4f21f142fd96fa55d8f128f673f2db4c46942" +dependencies = [ + "arrow", + "bigdecimal", + "chrono", + "crc32fast", + "datafusion", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", + "datafusion-functions-nested", + "datafusion-session", + "log", + "num-traits", + "percent-encoding", + "rand 0.9.5", + "serde_json", + "sha1 0.11.0", + "sha2 0.11.1", + "twox-hash", + "url", +] + +[[package]] +name = "datafusion-sql" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "816c224a0ef5469a32a562b5d4830544f714f97c5f50142109286d40a288c7db" +dependencies = [ + "arrow", + "bigdecimal", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-functions-nested", + "indexmap 2.14.2", + "log", + "recursive", + "regex", + "sqlparser", + "stacker", +] + +[[package]] +name = "datasketches" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" + +[[package]] +name = "defmt" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt-parser" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", + "unicode-xid", +] + +[[package]] +name = "destructure_traitobject" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c877555693c14d2f84191cfd3ad8582790fc52b5e2274b40b59cf5f5cea25c7" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fastnum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020d1b59a944bc239d79903fbac2eda2365138b44890c27979562f6592059dcd" +dependencies = [ + "bnum", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "ferroid" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee93edf3c501f0035bbeffeccfed0b79e14c311f12195ec0e661e114a0f60da4" +dependencies = [ + "futures", + "portable-atomic", + "rand 0.10.3", + "tokio", + "web-time", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags 2.13.2", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "h2" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.14.2", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hdfs-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18138c3dacdf0e95f834369f996b21d2b6fa99e71787ef45e2ece09b37bd1dd1" +dependencies = [ + "cc", + "java-locator", +] + +[[package]] +name = "hdrs" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1ac22889be3d758d8168395e2ab70374afb4b1959caa9586ab42e6dd9f454d" +dependencies = [ + "errno", + "hdfs-sys", + "libc", + "log", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "iceberg" +version = "0.10.1" +source = "git+https://github.com/apache/iceberg-rust?rev=1f3bc34058e3f3e45336bc5428f37b92649a608b#1f3bc34058e3f3e45336bc5428f37b92649a608b" +dependencies = [ + "aes-gcm", + "anyhow", + "apache-avro", + "array-init", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "arrow-string", + "as-any", + "async-trait", + "backon", + "base64 0.22.1", + "bimap", + "bytes", + "chrono", + "crc32fast", + "derive_builder", + "fastnum", + "flate2", + "fnv", + "form_urlencoded", + "futures", + "iceberg-property-macro", + "itertools 0.13.0", + "moka", + "murmur3", + "once_cell", + "ordered-float 4.6.0", + "parquet", + "rand 0.9.5", + "reqwest 0.12.28", + "roaring", + "serde", + "serde_bytes", + "serde_derive", + "serde_json", + "serde_repr", + "serde_with", + "strum 0.27.2", + "tokio", + "tracing", + "typed-builder", + "typetag", + "url", + "uuid", + "zeroize", + "zstd 0.13.3", +] + +[[package]] +name = "iceberg-property-macro" +version = "0.10.1" +source = "git+https://github.com/apache/iceberg-rust?rev=1f3bc34058e3f3e45336bc5428f37b92649a608b#1f3bc34058e3f3e45336bc5428f37b92649a608b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "iceberg-storage-opendal" +version = "0.10.1" +source = "git+https://github.com/apache/iceberg-rust?rev=1f3bc34058e3f3e45336bc5428f37b92649a608b#1f3bc34058e3f3e45336bc5428f37b92649a608b" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "cfg-if", + "futures", + "iceberg", + "iceberg-property-macro", + "opendal", + "reqsign-aws-v4", + "reqsign-core", + "serde", + "typetag", + "url", +] + +[[package]] +name = "icu_collections" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" + +[[package]] +name = "icu_properties" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" + +[[package]] +name = "icu_provider" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "inventory" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c" +dependencies = [ + "rustversion", +] + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "java-locator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09c46c1fe465c59b1474e665e85e1256c3893dd00927b8d55f63b09044c1e64f" +dependencies = [ + "glob", +] + +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "js-sys", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" +dependencies = [ + "spin 0.9.9", +] + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libbz2-rs-sys" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "liblzma" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa980ea6744074a5b5e0ac6b164aca94fc970121869b3e78ae12a2b0097dbea7" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c7e3581f367a78d7b7e7ae948d023310556f0cfc13156c2e4e00e25616492b9" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "link-section" +version = "0.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39c29a617ce3df32c08497bdc1ab6e2376e0b17948ac166a2fbe5977c5954cd9" + +[[package]] +name = "linktime-proc-macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e57c38c1e860fd37c604281cdfb1dd2216977fd76a50f85ba2f388ef3219616" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +dependencies = [ + "serde_core", +] + +[[package]] +name = "log-mdc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a94d21414c1f4a51209ad204c1776a3d0765002c76c6abcb602a6f09f1e881c7" + +[[package]] +name = "log4rs" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e947bb896e702c711fccc2bf02ab2abb6072910693818d1d6b07ee2b9dfd86c" +dependencies = [ + "anyhow", + "arc-swap", + "chrono", + "derive_more", + "fnv", + "humantime", + "libc", + "log", + "log-mdc", + "mock_instant", + "parking_lot", + "rand 0.9.5", + "serde", + "serde-value", + "serde_json", + "serde_yaml", + "thiserror", + "thread-id", + "typemap-ors", + "unicode-segmentation", + "winapi", +] + +[[package]] +name = "lru" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef9ac18847474e638e3702b76c65d4eb93428471a74778ef0f1be711717f89b5" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + +[[package]] +name = "lz4_flex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memory-stats" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c73f5c649995a115e1a0220b35e4df0a1294500477f97a91d0660fb5abeb574a" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "mock_instant" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb517913cfcfb9eeda59f36020269075a152701a01606c612f547e4890be399" + +[[package]] +name = "moka" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "murmur3" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" + +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint 0.4.8", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.8", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint 0.4.8", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "memchr", +] + +[[package]] +name = "object_store" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "chrono", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body-util", + "httparse", + "humantime", + "hyper", + "itertools 0.14.0", + "md-5 0.10.6", + "parking_lot", + "percent-encoding", + "quick-xml 0.39.4", + "rand 0.10.3", + "reqwest 0.12.28", + "ring", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "opendal" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33dbff14cc9bb085224256d6a81289d2f3202e85b06f408d42534b42162a4231" +dependencies = [ + "ctor", + "opendal-core", + "opendal-http-transport-reqwest", + "opendal-layer-concurrent-limit", + "opendal-layer-logging", + "opendal-layer-retry", + "opendal-layer-timeout", + "opendal-service-azdls", + "opendal-service-fs", + "opendal-service-gcs", + "opendal-service-oss", + "opendal-service-s3", +] + +[[package]] +name = "opendal-core" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48dbcef97d3eb7591db2c18d5cae95c836bcce07359b98d98dd6f4e861eb77b7" +dependencies = [ + "anyhow", + "asyncband 0.6.7", + "base64 0.23.1", + "bytes", + "futures", + "http", + "jiff", + "log", + "md-5 0.11.0", + "percent-encoding", + "quick-xml 0.41.0", + "reqsign-core", + "serde", + "serde_json", + "tokio", + "url", + "uuid", + "web-time", +] + +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85663452ea32bbc17e8f79ab29788c846d116ec7de31451be9c787e462dcb36c" +dependencies = [ + "bytes", + "futures", + "http", + "http-body", + "opendal-core", + "reqwest 0.13.5", +] + +[[package]] +name = "opendal-layer-concurrent-limit" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03f9e144b5228d741c3763ade8711d9b72e5fb6d998e779f2d7a09da0b5a3eba" +dependencies = [ + "asyncband 0.6.7", + "futures", + "http", + "opendal-core", +] + +[[package]] +name = "opendal-layer-logging" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2de17c61cd32e9d8d7d8efb91795e714dbccbafbc3c4e219e1542f4d6324161" +dependencies = [ + "log", + "opendal-core", +] + +[[package]] +name = "opendal-layer-retry" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94db301964a25366090484d61e6da16d5979cc8faf02c3e12210dc74fafed38" +dependencies = [ + "backon", + "log", + "opendal-core", +] + +[[package]] +name = "opendal-layer-timeout" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08956ddda07465449bfd48825f4f0f25e0351278ac974eaa659895d9d74f2c80" +dependencies = [ + "opendal-core", + "tokio", +] + +[[package]] +name = "opendal-service-azdls" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d564484a8f7d091827e825cfc91ed45bd48e64d262451ee041fe843db81bd8a" +dependencies = [ + "asyncband 0.6.7", + "base64 0.23.1", + "bytes", + "http", + "log", + "opendal-core", + "opendal-service-azure-common", + "quick-xml 0.41.0", + "reqsign-azure-storage", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "serde_json", +] + +[[package]] +name = "opendal-service-azure-common" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcc1bfdac4f54d9018c462dd32ad9e2f68fdf584f825c811550c8ffc87894cd" +dependencies = [ + "http", + "opendal-core", +] + +[[package]] +name = "opendal-service-fs" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7ef1e1c45f3f89282a59073897e0d685e51385fed0aea771714789525cff996" +dependencies = [ + "bytes", + "log", + "opendal-core", + "serde", + "tokio", + "xattr", +] + +[[package]] +name = "opendal-service-gcs" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da1f2a8c975fd22fea01f0409bcb8774f1ac02ad79863a3490098203121555d3" +dependencies = [ + "async-trait", + "bytes", + "http", + "log", + "opendal-core", + "percent-encoding", + "quick-xml 0.41.0", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-google", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "opendal-service-oss" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e3ce7a2ceb925e0f28f545b169eb57d04cec6116aae94b8584d2bdbe7d456c6" +dependencies = [ + "bytes", + "http", + "log", + "opendal-core", + "quick-xml 0.41.0", + "reqsign-aliyun-oss", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", +] + +[[package]] +name = "opendal-service-s3" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64335f9f24ccb62ac36f1d976342b48611a75ba61979813a4f78a4ebd94de42" +dependencies = [ + "base64 0.23.1", + "bytes", + "crc-fast", + "http", + "log", + "md-5 0.11.0", + "opendal-core", + "quick-xml 0.41.0", + "reqsign-aws-v4", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "url", +] + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordered-multimap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list", + "hashbrown 0.14.5", +] + +[[package]] +name = "ouroboros" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "parquet" +version = "59.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b3d2ef0fb136e9341570f219ecb168902e76079c6d81ad555f0a4840b1e08a3" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64 0.23.1", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.17.1", + "lz4_flex", + "num-bigint 0.5.1", + "num-integer", + "num-traits", + "object_store", + "parquet-variant", + "parquet-variant-compute", + "parquet-variant-json", + "ring", + "seq-macro", + "simdutf8", + "snap", + "tokio", + "twox-hash", + "zstd 0.13.3", +] + +[[package]] +name = "parquet-variant" +version = "59.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "587889e0dddd33f9eb536b20a489a20b2f556c4997ef1e09e49ae6df378d2ff0" +dependencies = [ + "arrow", + "arrow-schema", + "chrono", + "half", + "indexmap 2.14.2", + "num-traits", + "simdutf8", + "uuid", +] + +[[package]] +name = "parquet-variant-compute" +version = "59.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea15140dcafc6d1b4c17f1a12fcb9590ec8b82095cbd70925a5a8aa4c905e04a" +dependencies = [ + "arrow", + "arrow-schema", + "chrono", + "half", + "indexmap 2.14.2", + "parquet-variant", + "parquet-variant-json", + "serde_json", + "uuid", +] + +[[package]] +name = "parquet-variant-json" +version = "59.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf54eea151d1c8369a0b491fb727229149e4a09c9518e00aaaab1f816c25c8e" +dependencies = [ + "arrow-schema", + "base64 0.23.1", + "chrono", + "parquet-variant", + "serde_json", + "uuid", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest 0.10.7", + "hmac 0.12.1", +] + +[[package]] +name = "pem" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" +dependencies = [ + "base64 0.23.1", + "serde_core", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap 2.14.2", + "serde", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der", + "pbkdf2", + "scrypt", + "sha2 0.10.9", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "pkcs5", + "rand_core 0.6.4", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "prettyplease" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" +dependencies = [ + "proc-macro2", + "syn 3.0.9", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "version_check", + "yansi", +] + +[[package]] +name = "procfs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25485360a54d6861439d60facef26de713b1e126bf015ec8f98239467a2b82f7" +dependencies = [ + "bitflags 2.13.2", + "chrono", + "flate2", + "procfs-core", + "rustix", +] + +[[package]] +name = "procfs-core" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6401bf7b6af22f78b563665d15a22e9aef27775b79b149a66ca022468a4e405" +dependencies = [ + "bitflags 2.13.2", + "chrono", + "hex", +] + +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" +dependencies = [ + "heck 0.5.0", + "itertools 0.14.0", + "log", + "multimap", + "petgraph", + "prettyplease 0.2.37", + "prost", + "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + +[[package]] +name = "psm" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" +dependencies = [ + "ar_archive_writer", + "cc", +] + +[[package]] +name = "pulldown-cmark" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "278e9268c93b9e498a2da46c34d90ba93a3255eb99009646398f34bc73ab36eb" +dependencies = [ + "bitflags 2.13.2", + "memchr", + "unicase", +] + +[[package]] +name = "pulldown-cmark-to-cmark" +version = "22.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84bbb29c624230c4bd1047bbdb2aa47e41c860e9665ce62ba9504eebe91bf867" +dependencies = [ + "pulldown-cmark", +] + +[[package]] +name = "quad-rand" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" + +[[package]] +name = "quick-xml" +version = "0.39.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick-xml" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.3", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +dependencies = [ + "recursive-proc-macro-impl", + "stacker", +] + +[[package]] +name = "recursive-proc-macro-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqsign-aliyun-oss" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cf3d65d37ac910073d921c1cc58c3e4e6c963795c6062eee196e6d19054dde" +dependencies = [ + "anyhow", + "form_urlencoded", + "http", + "log", + "percent-encoding", + "reqsign-core", + "rust-ini", + "serde", + "serde_json", +] + +[[package]] +name = "reqsign-aws-core" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38c902d8ec745bd9eadd1abbcbe227f4c1fc2926b2ec8929f89234af3114500" +dependencies = [ + "bytes", + "form_urlencoded", + "hex", + "http", + "log", + "percent-encoding", + "quick-xml 0.42.0", + "reqsign-core", + "rust-ini", + "serde", + "serde_json", + "serde_urlencoded", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-aws-v4" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "906bfd028f96bfe73fdb892ce26f3b3f99b4f30feb9bd024be58bd6657f68b2b" +dependencies = [ + "bytes", + "http", + "log", + "quick-xml 0.42.0", + "reqsign-aws-core", + "reqsign-core", + "serde", +] + +[[package]] +name = "reqsign-azure-storage" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7260e6c1d89422ba62d624a32c491b561386fb364ac4bf0959c740e34b251a2e" +dependencies = [ + "anyhow", + "asyncband 0.7.3", + "base64 0.23.1", + "bytes", + "form_urlencoded", + "http", + "log", + "pem", + "percent-encoding", + "reqsign-core", + "rsa", + "serde", + "serde_json", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-core" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85fed85c436b6b920e4b36310f699a77771a91e38ae483df48447bd7d162671c" +dependencies = [ + "anyhow", + "asyncband 0.7.3", + "base64 0.23.1", + "bytes", + "hex", + "hmac 0.13.0", + "http", + "jiff", + "log", + "percent-encoding", + "rsa", + "serde", + "serde_json", + "sha1 0.11.0", + "sha2 0.11.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "reqsign-file-read-tokio" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "586a8e198057773f9616901fd723a4bc186812b9765bc0f20368661d6169d906" +dependencies = [ + "anyhow", + "reqsign-core", + "tokio", +] + +[[package]] +name = "reqsign-google" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719e55931cc81b8dcd69f95f0a2def55617cb76cb3aa2f8c439e10682e46fa10" +dependencies = [ + "bytes", + "form_urlencoded", + "http", + "log", + "percent-encoding", + "reqsign-aws-v4", + "reqsign-core", + "rsa", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http 0.6.11", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.4.2", + "web-sys", +] + +[[package]] +name = "reqwest" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http 0.6.11", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.5.0", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "roaring" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1" +dependencies = [ + "bytemuck", + "byteorder", +] + +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid 0.9.6", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "sha2 0.10.9", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rust-ini" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "aws-lc-rs", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a073f5dc7469f984c52ad2752b63b0807745133b6de880b7b64c1ac4c48aec4" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" + +[[package]] +name = "rustls-webpki" +version = "0.103.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cb23c424d8ee841bb4d813677ca237e2a610067d594fec130f87d6093495ab5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2 0.10.9", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde-big-array" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" +dependencies = [ + "serde", +] + +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float 2.10.1", + "serde", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "foldhash 0.2.0", + "indexmap 2.14.2", + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9adc193c780ef8f159aee8b61e2d5801aaa555e6eb0947fe45530ec506296f" +dependencies = [ + "base64 0.23.1", + "bs58", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.2", + "jiff", + "schemars 0.9.0", + "schemars 1.2.3", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e17bbc68e28663bbbb90df47e058aa7eda4fb445b89fe70457bb94fbccf6e49" +dependencies = [ + "darling 0.24.1", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap 2.14.2", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + +[[package]] +name = "snap" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlparser" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" +dependencies = [ + "log", + "recursive", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser_derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "stacker" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.61.2", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros 0.27.2", +] + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros 0.28.0", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "sysinfo" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "thread-id" +version = "5.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2010d27add3f3240c1fef7959f46c814487b216baee662af53be645ba7831c07" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tokio" +version = "1.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "tokio-util" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tonic" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1fa9b1a51188f3266a823208078315811269d06d272c3a61be0b9a194194ff" +dependencies = [ + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2", + "sync_wrapper", + "tokio", + "tokio-stream", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c07bd4e9f99e2a3ed8bebad596ae02a94e53745b68acc6900e22c3475c8bc44" +dependencies = [ + "prettyplease 0.2.37", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tonic-prost" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63c616ffaa648f7c5dd5657e97d9a285fcb44ff3b70da761d436f5a13ff7be6" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tonic-prost-build" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a05d8957fac6b3542999796cb4449d45da790751962f974a457e5e00c43d78c" +dependencies = [ + "prettyplease 0.2.37", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn 2.0.119", + "tempfile", + "tonic-build", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 2.14.2", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.13.2", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-http" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c" +dependencies = [ + "bitflags 2.13.2", + "bytes", + "http", + "percent-encoding", + "pin-project-lite", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "twox-hash" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2" +dependencies = [ + "rand 0.10.3", +] + +[[package]] +name = "typed-builder" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd9d30e3a08026c78f246b173243cf07b3696d274debd26680773b6773c2afc7" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "typemap-ors" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a68c24b707f02dd18f1e4ccceb9d49f2058c2fb86384ef9972592904d7a28867" +dependencies = [ + "unsafe-any-ors", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "typetag" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2" +dependencies = [ + "erased-serde", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "unicase" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "unsafe-any-ors" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a303d30665362d9680d7d91d78b23f5f899504d4f08b3c4cf08d055d87c0ad" +dependencies = [ + "destructure_traitobject", +] + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "url-escape" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd4105a4fc511f6b1c9616a372a1313ae7c15afcaac839617c1c7cf25ce0a965" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" +dependencies = [ + "js-sys", + "tokio", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "yoke" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe 7.3.0", +] + +[[package]] +name = "zstd" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" +dependencies = [ + "zstd-safe 8.2.0", +] + +[[package]] +name = "zstd-safe" +version = "7.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-safe" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.2+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/native/native-executors/Cargo.toml b/native/native-executors/Cargo.toml new file mode 100644 index 00000000000..117cd7e3df4 --- /dev/null +++ b/native/native-executors/Cargo.toml @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Comet Native Executors: the comet-scheduler and comet-executor binaries, built on Apache +# DataFusion Ballista. See https://github.com/apache/datafusion-comet/issues/7288. +# +# This is its own workspace, with its own lockfile, so that the main Comet build never depends +# on Ballista. It builds Comet's native crates by path together with the Ballista release on the +# same DataFusion major version. +[workspace] + +[package] +name = "comet-native-executors" +homepage = "https://datafusion.apache.org/comet" +repository = "https://github.com/apache/datafusion-comet" +authors = ["Apache DataFusion "] +description = "Apache DataFusion Comet: native executors and scheduler built on Apache DataFusion Ballista" +license = "Apache-2.0" +edition = "2021" +publish = false + +[dependencies] +arrow-flight = "59.2.0" +ballista-core = "55.0.0" +ballista-executor = { version = "55.0.0", default-features = false, features = ["arrow-ipc-optimizations"] } +ballista-scheduler = { version = "55.0.0", default-features = false } +datafusion = { version = "55.2.0", default-features = false } +# Comet's core crate, under the name of its library. Without the default hdfs-opendal feature, +# which links libjvm through libhdfs. +comet = { package = "datafusion-comet", path = "../core", default-features = false } +datafusion-comet-proto = { path = "../proto" } +datafusion-proto = "55.2.0" +env_logger = "0.11" +futures = "0.3" +log = "0.4" +prost = "0.14.3" +prost-types = "0.14.3" +tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync"] } +tonic = "0.14" + +[[bin]] +name = "comet-scheduler" +path = "src/bin/comet-scheduler.rs" + +[[bin]] +name = "comet-executor" +path = "src/bin/comet-executor.rs" diff --git a/native/native-executors/README.md b/native/native-executors/README.md new file mode 100644 index 00000000000..4224ab1de7a --- /dev/null +++ b/native/native-executors/README.md @@ -0,0 +1,87 @@ + + +# Comet Native Executors + +**Experimental.** Comet Native Executors run whole Spark queries without a JVM in the data plane. A +Spark driver plans each query with Catalyst and Comet, then submits it to a cluster of one +`comet-scheduler` and any number of `comet-executor` processes, built on +[Apache DataFusion Ballista](https://github.com/apache/datafusion-ballista). The design is in +[issue #7288](https://github.com/apache/datafusion-comet/issues/7288). + +This directory holds the cluster. The Spark driver side is in progress in +[#7289](https://github.com/apache/datafusion-comet/pull/7289). It is its own Cargo workspace, so the +main Comet build never depends on Ballista. + +## The scheduler's Flight service + +A client runs a query through the scheduler's Arrow Flight service: + +1. `GetFlightInfo`, with a `QueryRequest` (`native/proto/src/proto/native_executors.proto`) as the + command. The request is a stage graph of Comet native blocks connected by the exchanges between + them, stamped with the client's Comet version and plan format. The scheduler refuses a request + from another Comet build and plans each block with Comet's planner to check it. That builds no + object store for the block's files, so the scheduler uses no storage credentials. It then + builds a Ballista physical plan (`src/assembly.rs`), runs it on the Ballista scheduler it + wraps, and answers with one endpoint per result partition. +2. `DoGet` with each endpoint's ticket, at the endpoint's location: the executor that holds the + partition. A query's results stay there for `COMET_SCHEDULER_RESULT_RETENTION_SECONDS` after + the query ends (five minutes by default), whether or not a client has read them, and each + endpoint's expiration time says until when. + +A stage that several other stages read runs once for each of them, since Ballista's planner +doesn't share a stage between consumers yet. + +Each stage runs on the executors as a `CometFragmentExec` +(`native/core/src/execution/native_executors`), which plans the stage's block with Comet's planner +for each partition. Ballista moves the data between stages: + +| Exchange | Ballista plan | +| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| Hash | `RepartitionExec(Hash)`: Ballista's hash shuffle, with DataFusion's hash | +| Single partition, broadcast | `CoalescePartitionsExec` | +| Range | `CoalescePartitionsExec`, so a sort runs in one task ([ballista#2680](https://github.com/apache/datafusion-ballista/issues/2680)) | + +## Building and running + +```shell +CARGO_TARGET_DIR=native/target cargo build --manifest-path native/native-executors/Cargo.toml +native/target/debug/comet-scheduler & +native/target/debug/comet-executor & +``` + +Sharing `native/target` with the main workspace saves building the common dependencies twice. Both +binaries read their settings from environment variables, listed at the top of +`src/bin/comet-scheduler.rs` and `src/bin/comet-executor.rs`. + +## Testing + +```shell +CARGO_TARGET_DIR=native/target cargo test --manifest-path native/native-executors/Cargo.toml +``` + +`tests/assembly.rs` runs stage graphs on DataFusion alone. `tests/cluster.rs` starts the two +binaries and runs queries through the scheduler's Flight service, as a driver would. + +## Keeping it building + +This workspace builds Comet's native crates by path against the Ballista release on the same +DataFusion major version, and CI builds and tests it in the `linux-test-native-executors` job. When +Comet moves to a DataFusion major that Ballista hasn't released yet, the upgrade can turn that job +off until Ballista catches up. diff --git a/native/native-executors/src/assembly.rs b/native/native-executors/src/assembly.rs new file mode 100644 index 00000000000..8464d97fbc9 --- /dev/null +++ b/native/native-executors/src/assembly.rs @@ -0,0 +1,190 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Turns a driver's stage graph into a Ballista physical plan. + +use std::collections::HashMap; +use std::sync::Arc; + +use comet::execution::native_executors::{ + block_schema, partition_descriptor, CometFragmentExec, COMET_VERSION, +}; +use datafusion::common::{plan_datafusion_err, plan_err, Result}; +use datafusion::physical_expr::expressions::Column; +use datafusion::physical_expr::PhysicalExpr; +use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; +use datafusion::physical_plan::repartition::RepartitionExec; +use datafusion::physical_plan::ExecutionPlanProperties; +use datafusion::physical_plan::{ExecutionPlan, Partitioning}; +use datafusion_comet_proto::spark_expression::expr::ExprStruct; +use datafusion_comet_proto::spark_native_executors::{Fragment, QueryRequest, Stage, StageInput}; +use datafusion_comet_proto::spark_partitioning::partitioning::PartitioningStruct; +use datafusion_comet_proto::PROTO_FINGERPRINT; +use prost::Message; + +/// Checks that `request` comes from a driver running this Comet build. The plan format is not +/// yet stable across versions, and an executor would ignore any field it doesn't know. +pub fn check_version(request: &QueryRequest) -> Result<()> { + if request.comet_version != COMET_VERSION || request.proto_fingerprint != PROTO_FINGERPRINT { + return plan_err!( + "This Comet Native Executor cluster runs Comet {COMET_VERSION} with plan format \ + {PROTO_FINGERPRINT}, but the query comes from Comet {} with plan format {}. The \ + driver and the cluster must run the same Comet build.", + request.comet_version, + request.proto_fingerprint + ); + } + Ok(()) +} + +/// Builds the physical plan of `request`: a [`CometFragmentExec`] per stage, with a node that +/// Ballista's planner turns into a stage boundary on each edge. The root is the last stage. +/// +/// Each stage's block is planned with Comet's planner to check it before anything runs. That can +/// block, so don't call this on an async runtime's thread. A stage that several stages read runs +/// once for each of them, since Ballista's planner doesn't share a stage between consumers yet. +pub fn build_plan(request: QueryRequest) -> Result> { + let mut stages: Vec> = Vec::with_capacity(request.stages.len()); + for (index, stage) in request.stages.into_iter().enumerate() { + let fragment = build_stage(stage, &stages, &request.conf) + .map_err(|e| e.context(format!("Stage {index} of the query")))?; + stages.push(fragment); + } + stages + .pop() + .ok_or_else(|| plan_datafusion_err!("The query has no stages")) +} + +fn build_stage( + stage: Stage, + producers: &[Arc], + conf: &HashMap, +) -> Result> { + let block = stage + .plan + .as_ref() + .ok_or_else(|| plan_datafusion_err!("The stage has no plan"))? + .encode_to_vec(); + let mut inputs = Vec::with_capacity(stage.inputs.len()); + for input in &stage.inputs { + let producer = producers.get(input.producer as usize).ok_or_else(|| { + plan_datafusion_err!( + "The stage reads stage {}, which doesn't come before it", + input.producer + ) + })?; + inputs.push(exchange(input, Arc::clone(producer))?); + } + // A stage that reads exchanges has as many partitions as they deliver, which + // `CometFragmentExec` checks they agree on. + let num_partitions = stage + .inputs + .iter() + .zip(&inputs) + .find(|(input, _)| !input.broadcast) + .map(|(_, exchange)| exchange.output_partitioning().partition_count()) + .unwrap_or(stage.num_partitions as usize); + if num_partitions == 0 { + return plan_err!("The stage has no partitions"); + } + if !stage.partition_plans.is_empty() && stage.partition_plans.len() != num_partitions { + return plan_err!( + "The stage has {num_partitions} partitions but {} partition plans", + stage.partition_plans.len() + ); + } + // A stage that reads files carries its scans' schemas and options only in each partition's + // copy of the block, so check one of those. Checking reads none of its files. + let planned = block_schema(stage.partition_plans.first().unwrap_or(&block), conf)?; + if planned.fields().len() != stage.output_schema.len() { + return plan_err!( + "The stage's block produces {} columns but the stage declares {}", + planned.fields().len(), + stage.output_schema.len() + ); + } + + let mut children = vec![partition_descriptor( + num_partitions, + &stage.partition_plans, + )?]; + children.extend(inputs); + let fragment = Fragment { + // A stage that reads files runs each partition's own copy of the block instead. + block: if stage.partition_plans.is_empty() { + block + } else { + vec![] + }, + broadcast: stage.inputs.iter().map(|input| input.broadcast).collect(), + output_schema: stage.output_schema, + conf: conf.clone(), + }; + Ok(Arc::new(CometFragmentExec::try_new( + Arc::new(fragment), + children, + )?)) +} + +/// The exchange that carries `producer`'s output to a stage, as a node Ballista's planner turns +/// into a stage boundary. +/// +/// A hash exchange partitions with DataFusion's hash function, not Spark's Murmur3. That is +/// consistent within a query, which is all joins, aggregates and windows need, but a plan that +/// relies on Spark's own placement of rows, such as a join with a bucketed table, would be wrong. +fn exchange( + input: &StageInput, + producer: Arc, +) -> Result> { + if input.broadcast { + return Ok(Arc::new(CoalescePartitionsExec::new(producer))); + } + let partitioning = input + .partitioning + .as_ref() + .and_then(|p| p.partitioning_struct.as_ref()); + match partitioning { + Some(PartitioningStruct::HashPartition(hash)) => { + let schema = producer.schema(); + let keys = hash + .hash_expression + .iter() + .map(|expr| match expr.expr_struct.as_ref() { + Some(ExprStruct::Bound(bound)) + if (bound.index as usize) < schema.fields().len() => + { + let index = bound.index as usize; + Ok(Arc::new(Column::new(schema.field(index).name(), index)) + as Arc) + } + _ => plan_err!("Hash partitioning keys must be columns of the stage's input"), + }) + .collect::>>()?; + Ok(Arc::new(RepartitionExec::try_new( + producer, + Partitioning::Hash(keys, hash.num_partitions as usize), + )?)) + } + // Ballista can't plan a range shuffle for a physical plan yet + // (https://github.com/apache/datafusion-ballista/issues/2680), so the consuming stage + // reads a range exchange as one partition too. That sorts correctly, in a single task. + Some(PartitioningStruct::SinglePartition(_) | PartitioningStruct::RangePartition(_)) => { + Ok(Arc::new(CoalescePartitionsExec::new(producer))) + } + other => plan_err!("Comet Native Executors don't support this exchange yet: {other:?}"), + } +} diff --git a/native/native-executors/src/bin/comet-executor.rs b/native/native-executors/src/bin/comet-executor.rs new file mode 100644 index 00000000000..f1b7cec6a99 --- /dev/null +++ b/native/native-executors/src/bin/comet-executor.rs @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! A Comet executor: Ballista's executor, able to run the stages of Comet queries with Comet's +//! planner, operators and expressions. It needs no JVM. +//! +//! Configured through environment variables, with Ballista's defaults otherwise: +//! - `COMET_EXECUTOR_BIND_HOST` (default `127.0.0.1`) +//! - `COMET_EXECUTOR_EXTERNAL_HOST`: the host that drivers and other executors reach this +//! executor at (default: the address the scheduler sees it connect from) +//! - `COMET_EXECUTOR_PORT`: the Flight port that serves shuffle and result data (default `50051`) +//! - `COMET_EXECUTOR_GRPC_PORT` (default `50052`) +//! - `COMET_SCHEDULER_HOST` (default `localhost`) and `COMET_SCHEDULER_PORT` (default `50050`) +//! - `COMET_EXECUTOR_TASKS`: concurrent tasks (default: the number of CPUs) +//! - `COMET_EXECUTOR_WORK_DIR`: where shuffle and spill files go (default: a temporary directory) + +use std::sync::Arc; + +use ballista_executor::executor_process::{start_executor_process, ExecutorProcessConfig}; +use comet_native_executors::codec::CometPhysicalCodec; +use comet_native_executors::{env_var, init_logging}; + +#[tokio::main] +async fn main() -> Result<(), Box> { + init_logging(); + let mut config = ExecutorProcessConfig { + override_physical_codec: Some(Arc::new(CometPhysicalCodec::default())), + ..Default::default() + }; + if let Some(host) = env_var("COMET_EXECUTOR_BIND_HOST")? { + config.bind_host = host; + } + config.external_host = env_var("COMET_EXECUTOR_EXTERNAL_HOST")?; + if let Some(port) = env_var("COMET_EXECUTOR_PORT")? { + config.port = port; + } + if let Some(port) = env_var("COMET_EXECUTOR_GRPC_PORT")? { + config.grpc_port = port; + } + if let Some(host) = env_var("COMET_SCHEDULER_HOST")? { + config.scheduler_host = host; + } + if let Some(port) = env_var("COMET_SCHEDULER_PORT")? { + config.scheduler_port = port; + } + if let Some(tasks) = env_var("COMET_EXECUTOR_TASKS")? { + config.vcores = tasks; + } + config.work_dir = env_var("COMET_EXECUTOR_WORK_DIR")?; + + start_executor_process(Arc::new(config)).await?; + Ok(()) +} diff --git a/native/native-executors/src/bin/comet-scheduler.rs b/native/native-executors/src/bin/comet-scheduler.rs new file mode 100644 index 00000000000..d1506f335ac --- /dev/null +++ b/native/native-executors/src/bin/comet-scheduler.rs @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! A Comet scheduler: Ballista's scheduler, which Comet executors register with, wrapped in the +//! Arrow Flight service that Spark drivers submit queries to. +//! +//! Configured through environment variables, with Ballista's defaults otherwise: +//! - `COMET_SCHEDULER_BIND_HOST` (default `127.0.0.1`) +//! - `COMET_SCHEDULER_PORT`: Ballista's gRPC port, for executors (default `50050`) +//! - `COMET_SCHEDULER_FLIGHT_PORT`: the Flight port, for drivers (default `50060`) +//! - `COMET_SCHEDULER_RESULT_RETENTION_SECONDS`: how long a query's results stay readable after +//! the query ends, whether or not a driver has read them (default `300`). `0` keeps them. + +use std::net::SocketAddr; +use std::sync::Arc; + +use arrow_flight::flight_service_server::FlightServiceServer; +use ballista_scheduler::cluster::BallistaCluster; +use ballista_scheduler::config::SchedulerConfig; +use ballista_scheduler::scheduler_process::{create_scheduler, start_grpc_service}; +use comet_native_executors::codec::CometPhysicalCodec; +use comet_native_executors::frontend::CometFlightService; +use comet_native_executors::{env_var, init_logging}; +use datafusion_proto::protobuf::{LogicalPlanNode, PhysicalPlanNode}; +use log::info; + +#[tokio::main] +async fn main() -> Result<(), Box> { + init_logging(); + let mut config = SchedulerConfig { + override_physical_codec: Some(Arc::new(CometPhysicalCodec::default())), + ..Default::default() + }; + if let Some(host) = env_var("COMET_SCHEDULER_BIND_HOST")? { + config.bind_host = host; + } + if let Some(port) = env_var("COMET_SCHEDULER_PORT")? { + config.bind_port = port; + } + if let Some(seconds) = env_var("COMET_SCHEDULER_RESULT_RETENTION_SECONDS")? { + config.finished_job_data_clean_up_interval_seconds = seconds; + } + let flight_port: u16 = env_var("COMET_SCHEDULER_FLIGHT_PORT")?.unwrap_or(50060); + let grpc_addr: SocketAddr = format!("{}:{}", config.bind_host, config.bind_port).parse()?; + let flight_addr = SocketAddr::new(grpc_addr.ip(), flight_port); + let config = Arc::new(config); + config.validate()?; + + let cluster = BallistaCluster::new_from_config(&config).await?; + let scheduler = + create_scheduler::(cluster, Arc::clone(&config)).await?; + + info!("Comet scheduler: executors connect to {grpc_addr}, drivers to {flight_addr}"); + let grpc = tokio::spawn(start_grpc_service(grpc_addr, scheduler.clone())); + let flight = tonic::transport::Server::builder() + .add_service( + FlightServiceServer::new(CometFlightService::new(scheduler)) + .max_decoding_message_size(usize::MAX) + .max_encoding_message_size(usize::MAX), + ) + .serve(flight_addr); + tokio::select! { + result = grpc => result??, + result = flight => result?, + } + Ok(()) +} diff --git a/native/native-executors/src/codec.rs b/native/native-executors/src/codec.rs new file mode 100644 index 00000000000..b8b9a7f9320 --- /dev/null +++ b/native/native-executors/src/codec.rs @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Serializes Comet's operators inside the stage and task plans Ballista sends to executors. + +use std::sync::Arc; + +use ballista_core::serde::BallistaPhysicalExtensionCodec; +use comet::execution::native_executors::CometFragmentExec; +use datafusion::common::{internal_datafusion_err, Result}; +use datafusion::execution::TaskContext; +use datafusion::physical_plan::ExecutionPlan; +use datafusion_comet_proto::spark_native_executors::Fragment; +use datafusion_proto::physical_plan::{PhysicalExtensionCodec, PhysicalProtoConverterExtension}; +use prost::Message; + +/// Prefix of an encoded [`CometFragmentExec`], which tells it apart from Ballista's own nodes. +/// The rest is its `Fragment`; its children travel separately, as the plan's. +const FRAGMENT_MAGIC: &[u8] = b"COMET_FRAGMENT\0"; + +/// Ballista's physical extension codec, extended with Comet's operators. +#[derive(Debug, Default)] +pub struct CometPhysicalCodec { + ballista: BallistaPhysicalExtensionCodec, +} + +impl PhysicalExtensionCodec for CometPhysicalCodec { + fn try_decode( + &self, + buf: &[u8], + inputs: &[Arc], + ctx: &TaskContext, + proto_converter: &dyn PhysicalProtoConverterExtension, + ) -> Result> { + let Some(buf) = buf.strip_prefix(FRAGMENT_MAGIC) else { + return self.ballista.try_decode(buf, inputs, ctx, proto_converter); + }; + let fragment = Fragment::decode(buf) + .map_err(|e| internal_datafusion_err!("Cannot decode a CometFragmentExec: {e}"))?; + Ok(Arc::new(CometFragmentExec::try_new( + Arc::new(fragment), + inputs.to_vec(), + )?)) + } + + fn try_encode( + &self, + node: Arc, + buf: &mut Vec, + proto_converter: &dyn PhysicalProtoConverterExtension, + ) -> Result<()> { + let Some(fragment) = node.downcast_ref::() else { + return self.ballista.try_encode(node, buf, proto_converter); + }; + buf.extend_from_slice(FRAGMENT_MAGIC); + fragment + .fragment() + .encode(buf) + .map_err(|e| internal_datafusion_err!("Cannot encode a CometFragmentExec: {e}")) + } +} diff --git a/native/native-executors/src/frontend.rs b/native/native-executors/src/frontend.rs new file mode 100644 index 00000000000..b6077669819 --- /dev/null +++ b/native/native-executors/src/frontend.rs @@ -0,0 +1,259 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! The Arrow Flight service that Spark drivers submit queries to. +//! +//! `GetFlightInfo` takes a [`QueryRequest`] as its command, runs the query, and returns one +//! endpoint per result partition. Each endpoint names the executor that holds the partition and +//! carries an opaque ticket for its `DoGet`, so the driver never speaks Ballista's protocol. It +//! also says when the partition goes: Ballista deletes a query's results a set time after the +//! query ends, whether or not they have been read (see the comet-scheduler binary). + +use std::sync::Arc; +use std::time::{Duration, UNIX_EPOCH}; + +use arrow_flight::flight_service_server::FlightService; +use arrow_flight::{ + Action, ActionType, Criteria, Empty, FlightData, FlightDescriptor, FlightEndpoint, FlightInfo, + HandshakeRequest, HandshakeResponse, PollInfo, PutResult, SchemaResult, Ticket, +}; +use ballista_core::extension::SessionConfigExt; +use ballista_core::serde::protobuf::{job_status, PartitionLocation, SuccessfulJob}; +use ballista_core::serde::scheduler::{Action as BallistaAction, ShuffleFileKind}; +use ballista_scheduler::scheduler_server::SchedulerServer; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::prelude::SessionConfig; +use datafusion_comet_proto::spark_native_executors::QueryRequest; +use datafusion_proto::protobuf::{LogicalPlanNode, PhysicalPlanNode}; +use futures::stream::BoxStream; +use log::info; +use prost::Message; +use prost_types::Timestamp; +use tonic::{Request, Response, Status, Streaming}; + +use crate::assembly::{build_plan, check_version}; + +/// The Ballista session every query runs in. Every query uses Ballista's default settings, and +/// executors cache a runtime per session, so sharing one lets queries share those runtimes. +const SESSION_ID: &str = "comet"; + +/// The Flight service of a Comet scheduler, which runs queries on the Ballista scheduler it +/// wraps. +pub struct CometFlightService { + scheduler: SchedulerServer, +} + +impl CometFlightService { + pub fn new(scheduler: SchedulerServer) -> Self { + Self { scheduler } + } + + /// Runs `plan` to completion and returns its result: where its output partitions are, in + /// partition order, and when it ended. + async fn run(&self, plan: Arc) -> Result { + let ctx = self + .scheduler + .state + .session_manager + .create_or_update_session(SESSION_ID, &SessionConfig::new_with_ballista()) + .await + .map_err(|e| Status::internal(format!("Cannot create a session: {e}")))?; + let (sender, mut receiver) = tokio::sync::mpsc::channel(16); + let job_id = self + .scheduler + .submit_physical_plan("comet", ctx, plan, Some(sender)) + .await + .map_err(|e| Status::internal(format!("Cannot submit the query: {e}")))?; + info!("Submitted query as job {job_id}"); + while let Some(status) = receiver.recv().await { + match status.status { + Some(job_status::Status::Successful(mut job)) => { + job.partition_location.sort_by_key(|location| { + let partition = location.partition_id.as_ref().map(|p| p.partition_id); + (partition, location.map_partition_id) + }); + return Ok(job); + } + Some(job_status::Status::Failed(job)) => { + return Err(Status::internal(format!( + "Job {job_id} failed: {}", + job.error + ))) + } + _ => {} + } + } + Err(Status::internal(format!( + "Job {job_id} ended without a final status" + ))) + } + + /// When the executors delete the results of a query that ended at `ended_at`, in milliseconds + /// since the epoch. Ballista deletes them that long after the query ends, unless the scheduler + /// keeps them. Ballista starts that wait only after it records the end, so they are still + /// there until this time. + fn results_expire_at(&self, ended_at: u64) -> Option { + let retention = self + .scheduler + .state + .config + .finished_job_data_clean_up_interval_seconds; + if retention == 0 { + return None; + } + UNIX_EPOCH + .checked_add(Duration::from_millis(ended_at))? + .checked_add(Duration::from_secs(retention)) + .map(Timestamp::from) + } +} + +/// The Flight endpoint for a result partition: the executor's location, a ticket the executor's +/// Flight service reads as a Ballista partition fetch, and when the partition is deleted. +fn endpoint( + location: &PartitionLocation, + expires_at: Option, +) -> Result { + let executor = location + .executor_meta + .as_ref() + .ok_or_else(|| Status::internal("A result partition has no executor"))?; + let partition = location + .partition_id + .as_ref() + .ok_or_else(|| Status::internal("A result partition has no ID"))?; + let action = BallistaAction::FetchPartition { + job_id: partition.job_id.clone().into(), + stage_id: partition.stage_id as usize, + partition_id: partition.partition_id as usize, + host: executor.host.clone(), + port: executor.port as u16, + file_id: location.file_id, + layout: location.layout(), + file_kind: ShuffleFileKind::Data, + byte_ranges: vec![], + }; + let action: ballista_core::serde::protobuf::Action = action + .try_into() + .map_err(|e| Status::internal(format!("Cannot encode a partition fetch: {e}")))?; + let mut endpoint = FlightEndpoint::new() + .with_ticket(Ticket::new(action.encode_to_vec())) + .with_location(format!("grpc+tcp://{}:{}", executor.host, executor.port)); + endpoint.expiration_time = expires_at; + Ok(endpoint) +} + +fn unimplemented(method: &str) -> Result { + Err(Status::unimplemented(format!( + "A Comet scheduler doesn't implement {method}" + ))) +} + +#[tonic::async_trait] +impl FlightService for CometFlightService { + type HandshakeStream = BoxStream<'static, Result>; + type ListFlightsStream = BoxStream<'static, Result>; + type DoGetStream = BoxStream<'static, Result>; + type DoPutStream = BoxStream<'static, Result>; + type DoExchangeStream = BoxStream<'static, Result>; + type DoActionStream = BoxStream<'static, Result>; + type ListActionsStream = BoxStream<'static, Result>; + + async fn get_flight_info( + &self, + request: Request, + ) -> Result, Status> { + let descriptor = request.into_inner(); + let query = QueryRequest::decode(descriptor.cmd.as_ref()) + .map_err(|e| Status::invalid_argument(format!("Cannot decode the query: {e}")))?; + check_version(&query).map_err(|e| Status::failed_precondition(e.to_string()))?; + let plan = tokio::task::spawn_blocking(move || build_plan(query)) + .await + .map_err(|e| Status::internal(e.to_string()))? + .map_err(|e| Status::invalid_argument(e.to_string()))?; + let job = self.run(plan).await?; + let expires_at = self.results_expire_at(job.ended_at); + let mut info = FlightInfo::new(); + for location in &job.partition_location { + info = info.with_endpoint(endpoint(location, expires_at)?); + } + Ok(Response::new(info)) + } + + async fn handshake( + &self, + _request: Request>, + ) -> Result, Status> { + unimplemented("Handshake") + } + + async fn list_flights( + &self, + _request: Request, + ) -> Result, Status> { + unimplemented("ListFlights") + } + + async fn poll_flight_info( + &self, + _request: Request, + ) -> Result, Status> { + unimplemented("PollFlightInfo") + } + + async fn get_schema( + &self, + _request: Request, + ) -> Result, Status> { + unimplemented("GetSchema") + } + + async fn do_get( + &self, + _request: Request, + ) -> Result, Status> { + unimplemented("DoGet; fetch result partitions from the executors") + } + + async fn do_put( + &self, + _request: Request>, + ) -> Result, Status> { + unimplemented("DoPut") + } + + async fn do_exchange( + &self, + _request: Request>, + ) -> Result, Status> { + unimplemented("DoExchange") + } + + async fn do_action( + &self, + _request: Request, + ) -> Result, Status> { + unimplemented("DoAction") + } + + async fn list_actions( + &self, + _request: Request, + ) -> Result, Status> { + unimplemented("ListActions") + } +} diff --git a/native/native-executors/src/lib.rs b/native/native-executors/src/lib.rs new file mode 100644 index 00000000000..02251c83cb2 --- /dev/null +++ b/native/native-executors/src/lib.rs @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Comet Native Executors: a Comet scheduler and executors built on Apache DataFusion Ballista, +//! which run whole Spark queries planned by a Spark driver with no JVM. See +//! . +//! +//! A driver submits a query as a stage graph of Comet native blocks (a +//! [`QueryRequest`](datafusion_comet_proto::spark_native_executors::QueryRequest)) to the +//! scheduler's Arrow Flight service ([`frontend`]). The scheduler checks it, turns it into a +//! Ballista physical plan ([`assembly`]) and runs it on the executors, which rebuild each block +//! with Comet's planner. The driver then fetches the result partitions from the executors. + +use std::fmt::Display; +use std::str::FromStr; + +pub mod assembly; +pub mod codec; +pub mod frontend; + +/// The value of the environment variable `key`, if it is set, for the binaries' settings. +pub fn env_var(key: &str) -> Result, String> +where + T::Err: Display, +{ + std::env::var(key) + .ok() + .map(|value| { + value + .parse() + .map_err(|e| format!("Invalid {key} {value:?}: {e}")) + }) + .transpose() +} + +/// Logs at `info` unless `RUST_LOG` says otherwise. +pub fn init_logging() { + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); +} diff --git a/native/native-executors/tests/assembly.rs b/native/native-executors/tests/assembly.rs new file mode 100644 index 00000000000..63fcd103b9c --- /dev/null +++ b/native/native-executors/tests/assembly.rs @@ -0,0 +1,253 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Runs assembled stage graphs on DataFusion alone, without Ballista: each exchange runs as a +//! DataFusion repartition in this process. + +mod common; + +use comet_native_executors::assembly::build_plan; +use common::*; +use datafusion::arrow::datatypes::{DataType, Int32Type, Int64Type}; +use datafusion::physical_plan::{collect, ExecutionPlan, ExecutionPlanProperties}; +use datafusion::prelude::SessionContext; +use datafusion_comet_proto::spark_expression::{ + expr::ExprStruct, literal::Value, Expr, IfExpr, Literal, Subquery, +}; +use datafusion_comet_proto::spark_native_executors::{QueryRequest, Stage}; +use datafusion_comet_proto::spark_operator::{ + operator::OpStruct, NativeScan, NativeScanCommon, Operator, Projection, SparkFilePartition, + SparkPartitionedFile, +}; +use prost::Message; +use std::sync::Arc; + +async fn assemble(request: QueryRequest) -> datafusion::common::Result> { + tokio::task::spawn_blocking(move || build_plan(request)) + .await + .unwrap() +} + +/// Runs `request` and returns its single column, sorted. +async fn run(request: QueryRequest) -> Vec { + let plan = assemble(request).await.unwrap(); + let batches = collect(plan, SessionContext::new().task_ctx()) + .await + .unwrap(); + sorted_values::(&batches) +} + +#[tokio::test(flavor = "multi_thread")] +async fn leaf_stage_runs_every_partition() { + let request = query(vec![range_stage(100, 4)]); + assert_eq!(run(request).await, (0..100).collect::>()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn hash_exchange_moves_every_row() { + let request = query(vec![ + range_stage(100, 4), + reading_stage(0, hash_on_first_column(3), 3), + ]); + assert_eq!(run(request).await, (0..100).collect::>()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn single_partition_exchange_collects_every_row() { + let request = query(vec![ + range_stage(100, 4), + reading_stage(0, single_partition(), 1), + ]); + assert_eq!(run(request).await, (0..100).collect::>()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn range_exchange_is_read_as_one_partition() { + // The driver asks for the exchange's partition count. + let request = query(vec![ + range_stage(100, 4), + reading_stage(0, range_partitioning(4), 4), + ]); + let plan = assemble(request.clone()).await.unwrap(); + assert_eq!(plan.output_partitioning().partition_count(), 1); + assert_eq!(run(request).await, (0..100).collect::>()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn partition_plans_must_match_the_partitions() { + let mut stage = range_stage(100, 4); + let block = stage.plan.as_ref().unwrap().encode_to_vec(); + stage.partition_plans = vec![block.clone(), block]; + let error = assemble(query(vec![stage])).await.unwrap_err(); + assert!( + error + .to_string() + .contains("has 4 partitions but 2 partition plans"), + "{error}" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn reading_stage_has_as_many_partitions_as_its_exchange() { + // The partition count a driver gives a stage that reads an exchange doesn't matter. + let request = query(vec![ + range_stage(100, 4), + reading_stage(0, hash_on_first_column(3), 5), + ]); + let plan = assemble(request).await.unwrap(); + assert_eq!(plan.output_partitioning().partition_count(), 3); +} + +#[tokio::test(flavor = "multi_thread")] +async fn output_is_cast_to_the_types_the_stage_declares() { + // A range produces longs. A stage declaring an int column gets ints, as a Spark task's + // shuffle writer casts a column whose native type drifted from the one Spark declared. + let mut stage = range_stage(100, 4); + stage.output_schema = vec![field(int_type())]; + let plan = assemble(query(vec![stage])).await.unwrap(); + assert_eq!(plan.schema().field(0).data_type(), &DataType::Int32); + let batches = collect(plan, SessionContext::new().task_ctx()) + .await + .unwrap(); + assert_eq!( + sorted_values::(&batches), + (0..100).collect::>() + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn batches_have_the_schema_the_stage_declares() { + // `if(true, id, null)` has no nulls, but the block's output leaves it nullable. Spark can + // declare such a column not null, as it does a join key that a filter has checked, and a + // distributed engine checks each batch a stage sends against the stage's schema. + let literal = |value: Option, data_type| Expr { + expr_struct: Some(ExprStruct::Literal(Literal { + is_null: value.is_none(), + value, + datatype: Some(data_type), + })), + ..Default::default() + }; + let column = Expr { + expr_struct: Some(ExprStruct::If(Box::new(IfExpr { + if_expr: Some(Box::new(literal(Some(Value::BoolVal(true)), bool_type()))), + true_expr: Some(Box::new(bound(0, long_type()))), + false_expr: Some(Box::new(literal(None, long_type()))), + }))), + ..Default::default() + }; + let mut stage = range_stage(100, 4); + stage.plan = Some(Operator { + op_struct: Some(OpStruct::Projection(Projection { + project_list: vec![column], + })), + children: vec![stage.plan.take().unwrap()], + ..Default::default() + }); + stage.output_schema[0].nullable = false; + let plan = assemble(query(vec![stage])).await.unwrap(); + let batches = collect(Arc::clone(&plan), SessionContext::new().task_ctx()) + .await + .unwrap(); + assert!(batches.iter().all(|batch| batch.schema() == plan.schema())); + assert_eq!( + sorted_values::(&batches), + (0..100).collect::>() + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn declared_columns_must_match_the_block() { + let mut stage = range_stage(100, 4); + stage.output_schema = vec![field(long_type()), field(long_type())]; + let error = assemble(query(vec![stage])).await.unwrap_err(); + assert!( + error + .to_string() + .contains("produces 1 columns but the stage declares 2"), + "{error}" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn stage_without_partitions_is_rejected() { + let mut stage = range_stage(100, 4); + stage.num_partitions = 0; + let error = assemble(query(vec![stage])).await.unwrap_err(); + assert!(error.to_string().contains("no partitions"), "{error}"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn block_that_calls_into_the_jvm_is_rejected() { + // A scalar subquery reads its value from the JVM, which a native executor doesn't have. + let subquery = Expr { + expr_struct: Some(ExprStruct::Subquery(Subquery { + id: 1, + datatype: Some(long_type()), + })), + ..Default::default() + }; + let mut stage = range_stage(100, 4); + stage.plan = Some(Operator { + op_struct: Some(OpStruct::Projection(Projection { + project_list: vec![subquery], + })), + children: vec![stage.plan.take().unwrap()], + ..Default::default() + }); + let error = assemble(query(vec![stage])).await.unwrap_err(); + assert!(error.to_string().contains("calls into the JVM"), "{error}"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn file_stage_is_checked_without_its_object_store() { + // The scheduler checks a stage without reading its files, so it builds no object store and + // uses no storage credentials. Nothing can build a store for this URL, so only the stage's + // run finds that out. + let scan = |partitioned_file| Operator { + op_struct: Some(OpStruct::NativeScan(NativeScan { + common: Some(NativeScanCommon { + required_schema: vec![field(long_type())], + data_schema: vec![field(long_type())], + projection_vector: vec![0], + session_timezone: "UTC".to_string(), + ..Default::default() + }), + file_partition: Some(SparkFilePartition { partitioned_file }), + ..Default::default() + })), + ..Default::default() + }; + let file = SparkPartitionedFile { + file_path: "no-such-store://bucket/part-0.parquet".to_string(), + length: 16, + file_size: 16, + ..Default::default() + }; + let stage = Stage { + plan: Some(scan(vec![])), + partition_plans: vec![scan(vec![file]).encode_to_vec()], + num_partitions: 1, + output_schema: vec![field(long_type())], + ..Default::default() + }; + let plan = assemble(query(vec![stage])).await.unwrap(); + let error = collect(plan, SessionContext::new().task_ctx()) + .await + .unwrap_err(); + assert!(error.to_string().contains("no-such-store"), "{error}"); +} diff --git a/native/native-executors/tests/common/mod.rs b/native/native-executors/tests/common/mod.rs new file mode 100644 index 00000000000..5dc6b530cf7 --- /dev/null +++ b/native/native-executors/tests/common/mod.rs @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Stage graphs for the tests, built from Comet operators that need no files: `RangeScan` leaf +//! stages, and stages that return their input. + +// Each test file uses its own subset. +#![allow(dead_code)] + +use comet::execution::native_executors::COMET_VERSION; +use datafusion::arrow::array::{AsArray, RecordBatch}; +use datafusion::arrow::datatypes::ArrowPrimitiveType; +use datafusion_comet_proto::spark_expression::data_type::DataTypeId; +use datafusion_comet_proto::spark_expression::{self, expr::ExprStruct, BoundReference, Expr}; +use datafusion_comet_proto::spark_native_executors::{QueryRequest, Stage, StageInput}; +use datafusion_comet_proto::spark_operator::{ + operator::OpStruct, Operator, RangeScan, Scan, SparkStructField, +}; +use datafusion_comet_proto::spark_partitioning::{ + partitioning::PartitioningStruct, HashPartition, Partitioning, RangePartition, SinglePartition, +}; +use datafusion_comet_proto::PROTO_FINGERPRINT; + +/// A query of `stages` from a driver of this Comet build. +pub fn query(stages: Vec) -> QueryRequest { + QueryRequest { + comet_version: COMET_VERSION.to_string(), + proto_fingerprint: PROTO_FINGERPRINT.to_string(), + stages, + ..Default::default() + } +} + +fn data_type(type_id: DataTypeId) -> spark_expression::DataType { + spark_expression::DataType { + type_id: type_id as i32, + type_info: None, + } +} + +pub fn long_type() -> spark_expression::DataType { + data_type(DataTypeId::Int64) +} + +pub fn int_type() -> spark_expression::DataType { + data_type(DataTypeId::Int32) +} + +pub fn bool_type() -> spark_expression::DataType { + data_type(DataTypeId::Bool) +} + +/// Column `index` of an operator's input, of type `data_type`. +pub fn bound(index: i32, data_type: spark_expression::DataType) -> Expr { + Expr { + expr_struct: Some(ExprStruct::Bound(BoundReference { + index, + datatype: Some(data_type), + })), + ..Default::default() + } +} + +/// A nullable column `id` of type `data_type`, as a stage declares its output. +pub fn field(data_type: spark_expression::DataType) -> SparkStructField { + SparkStructField { + name: "id".to_string(), + data_type: Some(data_type), + nullable: true, + ..Default::default() + } +} + +/// A stage that runs `range(0, num_elements)` in `num_slices` partitions, like `spark.range`. +pub fn range_stage(num_elements: i64, num_slices: i32) -> Stage { + Stage { + plan: Some(Operator { + op_struct: Some(OpStruct::RangeScan(RangeScan { + start: 0, + step: 1, + num_elements, + num_slices, + })), + ..Default::default() + }), + num_partitions: num_slices as u32, + output_schema: vec![field(long_type())], + ..Default::default() + } +} + +/// A stage that returns what stage `producer` sends it through an exchange with `partitioning`. +/// `num_partitions` is the count the driver asks for, which is the exchange's. +pub fn reading_stage(producer: u32, partitioning: Partitioning, num_partitions: u32) -> Stage { + Stage { + plan: Some(Operator { + op_struct: Some(OpStruct::Scan(Scan { + fields: vec![long_type()], + source: "test".to_string(), + })), + ..Default::default() + }), + inputs: vec![StageInput { + producer, + partitioning: Some(partitioning), + broadcast: false, + }], + num_partitions, + output_schema: vec![field(long_type())], + ..Default::default() + } +} + +pub fn hash_on_first_column(num_partitions: i32) -> Partitioning { + Partitioning { + partitioning_struct: Some(PartitioningStruct::HashPartition(HashPartition { + hash_expression: vec![bound(0, long_type())], + num_partitions, + })), + } +} + +pub fn single_partition() -> Partitioning { + Partitioning { + partitioning_struct: Some(PartitioningStruct::SinglePartition(SinglePartition {})), + } +} + +/// A range exchange as a driver sends it: its sort keys don't matter here, and it has no bounds. +pub fn range_partitioning(num_partitions: i32) -> Partitioning { + Partitioning { + partitioning_struct: Some(PartitioningStruct::RangePartition(RangePartition { + num_partitions, + ..Default::default() + })), + } +} + +/// The values of the batches' single column, of type `T`, sorted. +pub fn sorted_values(batches: &[RecordBatch]) -> Vec +where + T::Native: Ord, +{ + let mut values: Vec = batches + .iter() + .flat_map(|batch| batch.column(0).as_primitive::().values().to_vec()) + .collect(); + values.sort_unstable(); + values +} From 7689fef4dd9100aa6298a43fac7041de9852269e Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 28 Oct 2026 09:26:09 -0600 Subject: [PATCH 04/36] test: run queries on comet-scheduler and comet-executor through Flight tests/cluster.rs starts the two binaries, submits stage graphs to the scheduler's Flight service as a driver would, and reads every result partition from the executors with DoGet. It covers a hash shuffle, a range exchange, and a query from another Comet build, which the scheduler refuses. --- native/native-executors/tests/cluster.rs | 265 +++++++++++++++++++++++ 1 file changed, 265 insertions(+) create mode 100644 native/native-executors/tests/cluster.rs diff --git a/native/native-executors/tests/cluster.rs b/native/native-executors/tests/cluster.rs new file mode 100644 index 00000000000..dee8dcaa7b5 --- /dev/null +++ b/native/native-executors/tests/cluster.rs @@ -0,0 +1,265 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Runs queries on a cluster of the comet-scheduler and comet-executor binaries, through the +//! scheduler's Arrow Flight service, as a Spark driver does. + +mod common; + +use std::future::Future; +use std::net::{TcpListener, TcpStream}; +use std::path::PathBuf; +use std::process::{Child, Command, Stdio}; +use std::sync::Mutex; +use std::time::{Duration, Instant, SystemTime}; + +use arrow_flight::error::FlightError; +use arrow_flight::{FlightClient, FlightDescriptor, FlightEndpoint, FlightInfo}; +use common::*; +use datafusion::arrow::array::RecordBatch; +use datafusion::arrow::datatypes::Int64Type; +use datafusion_comet_proto::spark_native_executors::QueryRequest; +use futures::TryStreamExt; +use prost::Message; +use tonic::transport::Endpoint; +use tonic::Code; + +/// A comet-scheduler and a comet-executor, stopped when dropped. +struct Cluster { + processes: Vec, + work_dir: PathBuf, + flight_url: String, +} + +/// Held while a cluster starts, so that two tests don't pick the same free port in between. +static STARTING: Mutex<()> = Mutex::new(()); + +impl Cluster { + fn start() -> Self { + Self::start_with(&[]) + } + + /// Starts a cluster whose scheduler also gets the settings `scheduler_env`. + fn start_with(scheduler_env: &[(&str, String)]) -> Self { + let _starting = STARTING.lock().unwrap_or_else(|e| e.into_inner()); + let scheduler_port = free_port(); + let flight_port = free_port(); + let executor_port = free_port(); + let executor_grpc_port = free_port(); + let mut cluster = Cluster { + processes: vec![], + work_dir: std::env::temp_dir().join(format!( + "comet-native-executors-{}-{flight_port}", + std::process::id() + )), + flight_url: format!("http://127.0.0.1:{flight_port}"), + }; + let mut env = vec![ + ("COMET_SCHEDULER_BIND_HOST", "127.0.0.1".to_string()), + ("COMET_SCHEDULER_PORT", scheduler_port.to_string()), + ("COMET_SCHEDULER_FLIGHT_PORT", flight_port.to_string()), + ]; + env.extend_from_slice(scheduler_env); + cluster.spawn(env!("CARGO_BIN_EXE_comet-scheduler"), &env); + wait_for(scheduler_port); + wait_for(flight_port); + let work_dir = cluster.work_dir.to_string_lossy().into_owned(); + cluster.spawn( + env!("CARGO_BIN_EXE_comet-executor"), + &[ + ("COMET_EXECUTOR_BIND_HOST", "127.0.0.1".to_string()), + ("COMET_EXECUTOR_PORT", executor_port.to_string()), + ("COMET_EXECUTOR_GRPC_PORT", executor_grpc_port.to_string()), + ("COMET_SCHEDULER_HOST", "127.0.0.1".to_string()), + ("COMET_SCHEDULER_PORT", scheduler_port.to_string()), + ("COMET_EXECUTOR_TASKS", "4".to_string()), + ("COMET_EXECUTOR_WORK_DIR", work_dir), + ], + ); + wait_for(executor_port); + cluster + } + + fn spawn(&mut self, binary: &str, env: &[(&str, String)]) { + let child = Command::new(binary) + .envs(env.iter().map(|(key, value)| (*key, value.as_str()))) + .env("RUST_LOG", "warn") + .stdout(Stdio::null()) + .spawn() + .unwrap_or_else(|e| panic!("Cannot start {binary}: {e}")); + self.processes.push(child); + } + + async fn get_flight_info(&self, request: &QueryRequest) -> Result { + let mut scheduler = connect(&self.flight_url).await; + within_time_limit( + scheduler.get_flight_info(FlightDescriptor::new_cmd(request.encode_to_vec())), + ) + .await + } + + /// Runs `request` and returns the batches of each result partition, in partition order. + async fn run(&self, request: &QueryRequest) -> Vec> { + let info = self.get_flight_info(request).await.unwrap(); + let mut partitions = vec![]; + for endpoint in &info.endpoint { + partitions.push(read(endpoint).await.unwrap()); + } + partitions + } +} + +/// Reads a result partition from the executor that holds it. +async fn read(endpoint: &FlightEndpoint) -> Result, FlightError> { + let mut executor = connect(&endpoint.location[0].uri).await; + let ticket = endpoint.ticket.clone().unwrap(); + within_time_limit(async { executor.do_get(ticket).await?.try_collect().await }).await +} + +impl Drop for Cluster { + fn drop(&mut self) { + // The executor first, so that it doesn't report to a scheduler that has gone. + for process in self.processes.iter_mut().rev() { + let _ = process.kill(); + let _ = process.wait(); + } + let _ = std::fs::remove_dir_all(&self.work_dir); + } +} + +/// Runs `call`, failing the test if it takes too long, as when no executor ever runs the query. +async fn within_time_limit(call: impl Future) -> T { + tokio::time::timeout(Duration::from_secs(300), call) + .await + .expect("The cluster took too long") +} + +/// Connects to a Flight location, such as an endpoint's `grpc+tcp://host:port`. +async fn connect(location: &str) -> FlightClient { + let uri = match location.strip_prefix("grpc+tcp://") { + Some(address) => format!("http://{address}"), + None => location.to_string(), + }; + let channel = Endpoint::from_shared(uri).unwrap().connect().await.unwrap(); + FlightClient::new(channel) +} + +fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() + .port() +} + +fn wait_for(port: u16) { + let deadline = Instant::now() + Duration::from_secs(60); + while TcpStream::connect(("127.0.0.1", port)).is_err() { + assert!( + Instant::now() < deadline, + "Nothing is listening on port {port}" + ); + std::thread::sleep(Duration::from_millis(100)); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn runs_a_query_across_a_hash_shuffle() { + let cluster = Cluster::start(); + let request = query(vec![ + range_stage(1000, 4), + reading_stage(0, hash_on_first_column(3), 3), + ]); + let partitions = cluster.run(&request).await; + assert_eq!(partitions.len(), 3); + assert_eq!( + sorted_values::(&partitions.concat()), + (0..1000).collect::>() + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn runs_a_query_across_a_range_exchange() { + let cluster = Cluster::start(); + let request = query(vec![ + range_stage(1000, 4), + reading_stage(0, range_partitioning(4), 4), + ]); + let partitions = cluster.run(&request).await; + assert_eq!(partitions.len(), 1); + assert_eq!( + sorted_values::(&partitions.concat()), + (0..1000).collect::>() + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn refuses_a_query_from_another_build() { + let cluster = Cluster::start(); + let mut request = query(vec![range_stage(10, 1)]); + request.comet_version = "0.0.0".to_string(); + match cluster.get_flight_info(&request).await { + Err(FlightError::Tonic(status)) => { + assert_eq!(status.code(), Code::FailedPrecondition, "{status}"); + assert!(status.message().contains("same Comet build"), "{status}"); + } + other => panic!("Expected the query to be refused, got {other:?}"), + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn endpoints_say_when_their_results_are_deleted() { + // A query's results stay readable for the scheduler's retention after the query ends, whether + // or not a client reads them, and each endpoint says until when. + let retention = Duration::from_secs(5); + let cluster = Cluster::start_with(&[( + "COMET_SCHEDULER_RESULT_RETENTION_SECONDS", + retention.as_secs().to_string(), + )]); + let submitted = SystemTime::now(); + let info = cluster + .get_flight_info(&query(vec![range_stage(100, 2)])) + .await + .unwrap(); + assert_eq!(info.endpoint.len(), 2); + let expiration = info.endpoint[0].expiration_time; + assert!(info + .endpoint + .iter() + .all(|endpoint| endpoint.expiration_time == expiration)); + let expires_at = SystemTime::try_from(expiration.expect("an expiration time")).unwrap(); + // The query ended between its submission and its answer. + assert!(expires_at >= submitted + retention); + assert!(expires_at <= SystemTime::now() + retention); + + // A client that reads one partition and then takes too long finds the other gone. + read(&info.endpoint[0]).await.unwrap(); + tokio::time::sleep( + expires_at + .duration_since(SystemTime::now()) + .unwrap_or_default(), + ) + .await; + let deadline = Instant::now() + Duration::from_secs(60); + while read(&info.endpoint[1]).await.is_ok() { + assert!( + Instant::now() < deadline, + "A result partition was still readable a minute after its endpoint expired" + ); + tokio::time::sleep(Duration::from_millis(200)).await; + } +} From 5dd751bd9c169fa061273250ea26b2fbde3b8d8d Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 28 Oct 2026 09:26:09 -0600 Subject: [PATCH 05/36] ci: build and test the native-executors workspace native/native-executors is a Cargo workspace of its own, so the Rust jobs don't cover it. The new linux-test-native-executors job runs clippy and the workspace's tests, including the ones that start a scheduler and an executor, and the lint job checks its formatting. --- .github/workflows/pr_build_linux.yml | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index 381df39aa95..aee487e5cda 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -106,6 +106,7 @@ jobs: run: | rustup component add rustfmt cd native && cargo fmt --all -- --check + cargo fmt --manifest-path native-executors/Cargo.toml --all -- --check # Fast syntactic-only scalafix check. Parses sources without compiling, so it # surfaces version-independent style issues (e.g. RedundantSyntax) in seconds, @@ -476,6 +477,36 @@ jobs: - name: Rust test steps uses: ./.github/actions/rust-test + # native/native-executors, the Comet Native Executors cluster (issue #7288), + # is a Cargo workspace of its own so that the main build never depends on + # Ballista. linux-test-rust doesn't build it, so this job does, and the lint + # job checks its formatting. When Comet moves to a DataFusion major that + # Ballista hasn't released yet, the upgrade can turn this job off until + # Ballista catches up. + linux-test-native-executors: + needs: lint + name: ubuntu-latest/native-executors + if: ${{ !inputs.cache-refresh-only && inputs.profiles != 'nightly' }} + runs-on: ubuntu-24.04 + container: + image: amd64/rust + steps: + - uses: actions/checkout@v7 + + - name: Setup Rust & Java toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: ${{ env.RUST_VERSION }} + jdk-version: 17 + + - name: Check Cargo clippy + run: | + cargo clippy --manifest-path native/native-executors/Cargo.toml --color=never \ + --all-targets -- -D warnings + + - name: Run Cargo test + run: cargo test --manifest-path native/native-executors/Cargo.toml --color=never + linux-test: # `lint` is already upstream via build-native; it is listed here so this # job can read its `profile-matrix` output. From 6cc27ee7b7c939ecf68e6abd0d7a1bfc3d9a3b5d Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 28 Oct 2026 04:14:28 -0600 Subject: [PATCH 06/36] feat: run queries on Comet Native Executors from a Spark driver With spark.comet.nativeExecutors.enabled, CometRule replaces the plan Comet would execute with a CometNativeExecutorsExec leaf, which runs the whole query on the cluster at spark.comet.nativeExecutors.scheduler.url. This works with AQE on, where Comet's query-stage-prep rule sees the whole plan, and off. The leaf is columnar, so Spark Connect's execute-and-submitJob path and collect() both read it through local tasks on the driver, each of which fetches one result partition. The stage graph builder turns each native block into a stage and each native shuffle into an input, and refuses, naming the operator, any query that cannot run entirely natively, including a block whose plan calls back into the JVM. Operators that only turn the result into JVM objects, as Dataset.rdd adds, stay on the driver. The driver's Flight client is a JNI library in libcomet behind the native-executors Cargo feature, off by default. CometNativeExecutorsSuite starts a scheduler and an executor and checks queries against Spark; it runs when COMET_NATIVE_EXECUTORS_BIN_DIR names the binaries. See #7288. --- native/Cargo.lock | 143 +++++++ native/core/Cargo.toml | 6 + native/core/src/execution/jni_api.rs | 2 +- .../src/execution/native_executors/client.rs | 179 +++++++++ .../src/execution/native_executors/mod.rs | 2 + native/proto/src/proto/native_executors.proto | 14 + .../scala/org/apache/comet/CometConf.scala | 22 ++ .../apache/comet/NativeExecutorsClient.scala | 62 +++ .../org/apache/comet/rules/CometRule.scala | 5 + .../sql/comet/CometNativeExecutorsExec.scala | 369 ++++++++++++++++++ .../exec/CometNativeExecutorsSuite.scala | 231 +++++++++++ 11 files changed, 1034 insertions(+), 1 deletion(-) create mode 100644 native/core/src/execution/native_executors/client.rs create mode 100644 spark/src/main/scala/org/apache/comet/NativeExecutorsClient.scala create mode 100644 spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala create mode 100644 spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala diff --git a/native/Cargo.lock b/native/Cargo.lock index 673325aedfe..670961ad86e 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -347,6 +347,27 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-flight" +version = "59.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b543210778ac15d5f7a10153dc39e099f1eee756363589a842029e6c5c76e423" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "base64 0.23.1", + "bytes", + "futures", + "prost", + "prost-types", + "tonic", + "tonic-prost", +] + [[package]] name = "arrow-ipc" version = "59.3.1" @@ -1033,6 +1054,49 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + [[package]] name = "backon" version = "1.6.0" @@ -1979,6 +2043,7 @@ name = "datafusion-comet" version = "1.3.0" dependencies = [ "arrow", + "arrow-flight", "assertables", "async-trait", "aws-config", @@ -2044,6 +2109,7 @@ dependencies = [ "tikv-jemalloc-ctl", "tikv-jemallocator", "tokio", + "tonic", "typetag", "url", "uuid", @@ -3465,6 +3531,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "humantime" version = "2.4.0" @@ -3494,6 +3566,7 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -3517,6 +3590,19 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.21" @@ -4241,6 +4327,12 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "md-5" version = "0.10.6" @@ -4295,6 +4387,12 @@ dependencies = [ "libmimalloc-sys", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -6883,6 +6981,47 @@ dependencies = [ "tokio", ] +[[package]] +name = "tonic" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1fa9b1a51188f3266a823208078315811269d06d272c3a61be0b9a194194ff" +dependencies = [ + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2", + "sync_wrapper", + "tokio", + "tokio-stream", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-prost" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63c616ffaa648f7c5dd5657e97d9a285fcb44ff3b70da761d436f5a13ff7be6" +dependencies = [ + "bytes", + "prost", + "tonic", +] + [[package]] name = "tower" version = "0.5.3" @@ -6891,11 +7030,15 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", + "indexmap 2.14.2", "pin-project-lite", + "slab", "sync_wrapper", "tokio", + "tokio-util", "tower-layer", "tower-service", + "tracing", ] [[package]] diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml index 254e0740e97..c84c09492a2 100644 --- a/native/core/Cargo.toml +++ b/native/core/Cargo.toml @@ -54,6 +54,9 @@ async-trait = { workspace = true } log = "0.4" log4rs = "1.4.0" prost = "0.14.3" +# The driver's client of a Comet Native Executor cluster; see the native-executors feature. +arrow-flight = { version = "59.2.0", optional = true } +tonic = { version = "0.14", optional = true } jni = { workspace = true } rand = { workspace = true } num = { workspace = true } @@ -132,6 +135,9 @@ python-udf = ["dep:pyo3", "dep:libc"] contrib-lance = ["dep:comet-contrib-lance"] hdfs-opendal = ["opendal/services-hdfs", "object_store_opendal", "hdfs-sys"] jemalloc = ["tikv-jemallocator", "tikv-jemalloc-ctl"] +# The JNI client a Spark driver uses to run queries on Comet Native Executors (issue #7288). Off +# by default while the feature is experimental, so the standard build links no gRPC client. +native-executors = ["dep:arrow-flight", "dep:tonic"] # Native Delta Lake scan support for the JVM-planned path (contrib/delta-spark). # In the default set so trying the contrib needs only the jar and the config, # not a custom native build. It is inert at runtime unless the contrib jar is diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index d915abfed0b..33c594d9ca8 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -1233,7 +1233,7 @@ fn prepare_output( /// # Safety /// Each address must point to a writable `FFI_ArrowArray` / `FFI_ArrowSchema` the caller owns. /// Whatever the structs hold is overwritten without being released. -unsafe fn export_batch( +pub(crate) unsafe fn export_batch( array_addrs: &[i64], schema_addrs: &[i64], output_batch: RecordBatch, diff --git a/native/core/src/execution/native_executors/client.rs b/native/core/src/execution/native_executors/client.rs new file mode 100644 index 00000000000..d268316070e --- /dev/null +++ b/native/core/src/execution/native_executors/client.rs @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! The JNI client a Spark driver uses to run queries on a Comet Native Executor cluster: the +//! native methods of `org.apache.comet.NativeExecutorsClient`. +//! +//! The driver submits a query to the scheduler's Arrow Flight service, which answers with one +//! endpoint per result partition, and then reads each partition from its executor with `DoGet`. + +use arrow_flight::decode::FlightRecordBatchStream; +use arrow_flight::flight_service_client::FlightServiceClient; +use arrow_flight::{FlightClient, FlightDescriptor, Ticket}; +use datafusion_comet_proto::spark_native_executors::{ + QueryRequest, ResultEndpoint, ResultEndpoints, +}; +use datafusion_comet_proto::PROTO_FINGERPRINT; +use futures::StreamExt; +use jni::objects::{JByteArray, JClass, JLongArray, JString, ReleaseMode}; +use jni::sys::{jbyteArray, jlong}; +use jni::EnvUnowned; +use prost::Message; +use tonic::transport::{Channel, Endpoint}; + +use super::COMET_VERSION; +use crate::errors::{try_unwrap_or_throw, CometError, CometResult}; +use crate::execution::jni_api::{export_batch, get_runtime}; + +fn client_error(context: &str, error: impl std::fmt::Display) -> CometError { + CometError::Internal(format!("Comet Native Executors: {context}: {error}")) +} + +/// Connects to the Flight service at `location`, a Flight location such as +/// `grpc+tcp://host:port` or an `http://` URL. +async fn connect(location: &str) -> CometResult { + let uri = match location.strip_prefix("grpc+tcp://") { + Some(address) => format!("http://{address}"), + None => location.to_string(), + }; + let channel: Channel = Endpoint::from_shared(uri) + .map_err(|e| client_error(&format!("invalid location {location}"), e))? + .connect() + .await + .map_err(|e| client_error(&format!("cannot connect to {location}"), e))?; + // Query plans and result batches can be larger than gRPC's default 4 MiB message limit. + let inner = FlightServiceClient::new(channel) + .max_decoding_message_size(usize::MAX) + .max_encoding_message_size(usize::MAX); + Ok(FlightClient::new_from_inner(inner)) +} + +/// Submits `query`, a serialized `QueryRequest`, to the scheduler at `scheduler_url` and waits +/// for it to finish. Returns where its result partitions are. +async fn submit(scheduler_url: &str, query: &[u8]) -> CometResult { + let mut request = QueryRequest::decode(query).map_err(|e| client_error("bad query", e))?; + request.comet_version = COMET_VERSION.to_string(); + request.proto_fingerprint = PROTO_FINGERPRINT.to_string(); + let mut client = connect(scheduler_url).await?; + let info = client + .get_flight_info(FlightDescriptor::new_cmd(request.encode_to_vec())) + .await + .map_err(|e| client_error("the query failed", e))?; + let endpoints = info + .endpoint + .into_iter() + .map(|endpoint| { + let location = endpoint + .location + .first() + .map(|location| location.uri.clone()) + .ok_or_else(|| client_error("bad result", "a partition has no location"))?; + let ticket = endpoint + .ticket + .map(|ticket| ticket.ticket.to_vec()) + .ok_or_else(|| client_error("bad result", "a partition has no ticket"))?; + Ok(ResultEndpoint { location, ticket }) + }) + .collect::>>()?; + Ok(ResultEndpoints { endpoints }) +} + +/// Runs a query: `query` is a serialized `QueryRequest`, which this fills in with the driver's +/// Comet version and plan format. Returns a serialized `ResultEndpoints`. +#[no_mangle] +pub extern "system" fn Java_org_apache_comet_NativeExecutorsClient_submit( + e: EnvUnowned, + _class: JClass, + scheduler_url: JString, + query: JByteArray, +) -> jbyteArray { + try_unwrap_or_throw(&e, |env| { + let scheduler_url = scheduler_url + .try_to_string(env) + .map_err(|e| client_error("bad scheduler URL", e))?; + let query = env.convert_byte_array(query)?; + let endpoints = get_runtime().block_on(submit(&scheduler_url, &query))?; + Ok(env + .byte_array_from_slice(&endpoints.encode_to_vec())? + .into_raw()) + }) +} + +/// Starts reading a result partition from the executor at `location`. Returns a handle for +/// `nextBatch` and `closePartition`. +#[no_mangle] +pub extern "system" fn Java_org_apache_comet_NativeExecutorsClient_openPartition( + e: EnvUnowned, + _class: JClass, + location: JString, + ticket: JByteArray, +) -> jlong { + try_unwrap_or_throw(&e, |env| { + let location = location + .try_to_string(env) + .map_err(|e| client_error("bad location", e))?; + let ticket = env.convert_byte_array(ticket)?; + let stream = get_runtime().block_on(async { + connect(&location) + .await? + .do_get(Ticket::new(ticket)) + .await + .map_err(|e| client_error(&format!("cannot read a partition from {location}"), e)) + })?; + Ok(Box::into_raw(Box::new(stream)) as jlong) + }) +} + +/// Moves the partition's next batch into the Arrow C Data Interface structs at `array_addrs` +/// and `schema_addrs`, one pair per column. Returns its row count, or -1 at the end. +#[no_mangle] +pub extern "system" fn Java_org_apache_comet_NativeExecutorsClient_nextBatch( + e: EnvUnowned, + _class: JClass, + handle: jlong, + array_addrs: JLongArray, + schema_addrs: JLongArray, +) -> jlong { + try_unwrap_or_throw(&e, |env| { + // SAFETY: `handle` comes from `openPartition` and hasn't been closed. + let stream = unsafe { &mut *(handle as *mut FlightRecordBatchStream) }; + let Some(batch) = get_runtime().block_on(stream.next()) else { + return Ok(-1); + }; + let batch = batch.map_err(|e| client_error("cannot read a partition", e))?; + let array_addrs = unsafe { array_addrs.get_elements(env, ReleaseMode::NoCopyBack)? }; + let schema_addrs = unsafe { schema_addrs.get_elements(env, ReleaseMode::NoCopyBack)? }; + // SAFETY: the JVM allocated one array and one schema struct per column. + unsafe { export_batch(&array_addrs, &schema_addrs, batch, false) } + }) +} + +/// Stops reading a partition and releases its handle. +#[no_mangle] +pub extern "system" fn Java_org_apache_comet_NativeExecutorsClient_closePartition( + e: EnvUnowned, + _class: JClass, + handle: jlong, +) { + try_unwrap_or_throw(&e, |_env| { + if handle != 0 { + // SAFETY: `handle` comes from `openPartition`, and the JVM closes it once. + drop(unsafe { Box::from_raw(handle as *mut FlightRecordBatchStream) }); + } + Ok(()) + }) +} diff --git a/native/core/src/execution/native_executors/mod.rs b/native/core/src/execution/native_executors/mod.rs index fc0b9d6b2f4..f7c152d3fba 100644 --- a/native/core/src/execution/native_executors/mod.rs +++ b/native/core/src/execution/native_executors/mod.rs @@ -24,6 +24,8 @@ //! partition of a stage, reading its inputs from other DataFusion plans, such as a distributed //! engine's shuffle readers, instead of from the JVM. +#[cfg(feature = "native-executors")] +mod client; mod fragment; pub use fragment::{block_schema, partition_descriptor, CometFragmentExec}; diff --git a/native/proto/src/proto/native_executors.proto b/native/proto/src/proto/native_executors.proto index 6c0ba50afb4..a324dc690a1 100644 --- a/native/proto/src/proto/native_executors.proto +++ b/native/proto/src/proto/native_executors.proto @@ -80,3 +80,17 @@ message StageInput { // Every partition of this stage reads the producer's whole output. bool broadcast = 3; } + +// The result partitions of a query, in order. The driver's native library converts the +// endpoints of the FlightInfo the scheduler returns into this message for the JVM. +message ResultEndpoints { + repeated ResultEndpoint endpoints = 1; +} + +message ResultEndpoint { + // Arrow Flight location of the executor that holds the partition, such as + // grpc+tcp://host:port. + string location = 1; + // Opaque ticket to pass to DoGet. + bytes ticket = 2; +} diff --git a/spark/src/main/scala/org/apache/comet/CometConf.scala b/spark/src/main/scala/org/apache/comet/CometConf.scala index 220d6f6b62a..8988943189f 100644 --- a/spark/src/main/scala/org/apache/comet/CometConf.scala +++ b/spark/src/main/scala/org/apache/comet/CometConf.scala @@ -508,6 +508,28 @@ object CometConf extends ShimCometConf { .checkValue(v => v > 0 && v <= Int.MaxValue, "Must be positive and fit in an Int.") .createWithDefault(1024 * 1024) + val COMET_NATIVE_EXECUTORS_ENABLED: ConfigEntry[Boolean] = + conf("spark.comet.nativeExecutors.enabled") + .category(CATEGORY_EXEC) + .doc( + "Experimental. When enabled, Comet runs each query on a Comet Native Executor cluster " + + "instead of on Spark's executors: the driver plans the query and submits it to the " + + "cluster's scheduler (`spark.comet.nativeExecutors.scheduler.url`), and the results " + + "come back to the driver. A query that cannot run entirely natively fails, naming " + + "the operator that cannot. See https://github.com/apache/datafusion-comet/issues/7288.") + .internal() + .booleanConf + .createWithDefault(false) + + val COMET_NATIVE_EXECUTORS_SCHEDULER_URL: ConfigEntry[String] = + conf("spark.comet.nativeExecutors.scheduler.url") + .category(CATEGORY_EXEC) + .doc("Experimental. The Arrow Flight URL of a Comet Native Executor cluster's scheduler, " + + "such as `http://host:50060`. See `spark.comet.nativeExecutors.enabled`.") + .internal() + .stringConf + .createWithDefault("http://localhost:50060") + val COMET_NATIVE_COLUMNAR_TO_ROW_ENABLED: ConfigEntry[Boolean] = conf(s"$COMET_EXEC_CONFIG_PREFIX.columnarToRow.native.enabled") .category(CATEGORY_EXEC) diff --git a/spark/src/main/scala/org/apache/comet/NativeExecutorsClient.scala b/spark/src/main/scala/org/apache/comet/NativeExecutorsClient.scala new file mode 100644 index 00000000000..fc4d3c98d9c --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/NativeExecutorsClient.scala @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet + +/** + * The JNI client a Spark driver uses to run queries on a Comet Native Executor cluster. See + * https://github.com/apache/datafusion-comet/issues/7288. + * + * The native methods exist only in a libcomet built with the `native-executors` Cargo feature. + */ +class NativeExecutorsClient extends NativeBase { + + /** + * Runs a query on the cluster and waits for it to finish. + * + * @param schedulerUrl + * the Arrow Flight URL of the cluster's scheduler + * @param query + * a serialized `QueryRequest`, whose Comet version and plan format the native library fills + * in + * @return + * a serialized `ResultEndpoints`: where each result partition is, in order + */ + @native def submit(schedulerUrl: String, query: Array[Byte]): Array[Byte] + + /** + * Starts reading a result partition. + * + * @return + * a handle for [[nextBatch]] and [[closePartition]] + */ + @native def openPartition(location: String, ticket: Array[Byte]): Long + + /** + * Moves the partition's next batch into the Arrow C Data Interface structs at `arrayAddrs` and + * `schemaAddrs`, one pair per column. + * + * @return + * the batch's row count, or -1 at the end of the partition + */ + @native def nextBatch(handle: Long, arrayAddrs: Array[Long], schemaAddrs: Array[Long]): Long + + /** Stops reading a partition and releases its handle. */ + @native def closePartition(handle: Long): Unit +} diff --git a/spark/src/main/scala/org/apache/comet/rules/CometRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometRule.scala index fbde4a79090..07b2eea6863 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometRule.scala @@ -25,6 +25,7 @@ import scala.util.control.NonFatal import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.trees.TreeNodeTag +import org.apache.spark.sql.comet.CometNativeExecutors import org.apache.spark.sql.execution.{ApplyColumnarRulesAndInsertTransitions, BaseSubqueryExec, ColumnarToRowExec, ExecSubqueryExpression, InputAdapter, QueryExecution, ReusedSubqueryExec, RowToColumnarExec, SparkPlan, WholeStageCodegenExec} import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, InsertAdaptiveSparkPlan, QueryStageExec} import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, Exchange} @@ -163,6 +164,10 @@ case class CometRule(session: SparkSession, queryStagePrep: Boolean = false) if (CometRule.planOnlyApplies(conf, plan)) { reportPlanOnlyCoverage(plan) plan + } else if (CometNativeExecutors.applies(conf, plan, queryStagePrep)) { + // The whole query runs on a Comet Native Executor cluster. The conversion alone leaves + // every native block serialized, and nothing reverts a stage to Spark. + CometNativeExecutors.offload(plan, convert(plan)) } else { convert(plan) } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala new file mode 100644 index 00000000000..ef9c4de17c5 --- /dev/null +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala @@ -0,0 +1,369 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.sql.comet + +import java.util.IdentityHashMap + +import scala.collection.mutable.ArrayBuffer +import scala.jdk.CollectionConverters._ + +import org.apache.spark.{Partition, SparkContext, TaskContext} +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, SinglePartition} +import org.apache.spark.sql.comet.execution.shuffle.{CometNativeShuffle, CometShuffleExchangeExec} +import org.apache.spark.sql.execution.{ColumnarToRowExec, DeserializeToObjectExec, LeafExecNode, SparkPlan} +import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, QueryStageExec} +import org.apache.spark.sql.execution.command.{DataWritingCommandExec, ExecutedCommandExec} +import org.apache.spark.sql.execution.datasources.v2.V2CommandExec +import org.apache.spark.sql.execution.exchange.Exchange +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.StructField +import org.apache.spark.sql.vectorized.ColumnarBatch + +import com.google.protobuf.Descriptors.FieldDescriptor +import com.google.protobuf.Message + +import org.apache.comet.{CometConf, CometExecIterator, CometExplainInfo, CometRuntimeException, NativeExecutorsClient} +import org.apache.comet.serde.{PartitioningOuterClass, QueryPlanSerde} +import org.apache.comet.serde.Config.ConfigMap +import org.apache.comet.serde.NativeExecutors.{QueryRequest, ResultEndpoints, Stage, StageInput} +import org.apache.comet.serde.OperatorOuterClass.Operator +import org.apache.comet.serde.operator.schema2Proto +import org.apache.comet.vector.NativeUtil + +/** + * Runs a whole query on a Comet Native Executor cluster instead of on Spark's executors. See + * https://github.com/apache/datafusion-comet/issues/7288. + * + * This leaf replaces the plan Comet would otherwise execute. `doExecuteColumnar` submits the + * query to the cluster's scheduler and waits for it on the driver; each partition of the RDD it + * returns then reads one result partition from the executor that holds it. + * + * @param query + * the serialized `QueryRequest` built by [[CometNativeExecutors.buildQuery]] + */ +case class CometNativeExecutorsExec( + override val output: Seq[Attribute], + query: Array[Byte], + schedulerUrl: String) + extends LeafExecNode { + + override def supportsColumnar: Boolean = true + + override protected def doExecute(): RDD[InternalRow] = + throw new UnsupportedOperationException(s"$nodeName only produces columnar output") + + override protected def doExecuteColumnar(): RDD[ColumnarBatch] = { + val endpoints = + ResultEndpoints.parseFrom(new NativeExecutorsClient().submit(schedulerUrl, query)) + val partitions = endpoints.getEndpointsList.asScala.map { endpoint => + (endpoint.getLocation, endpoint.getTicket.toByteArray) + } + new CometNativeExecutorsRDD(sparkContext, partitions.toSeq, output.length) + } + + override def stringArgs: Iterator[Any] = + Iterator(output, s"${query.length} bytes", schedulerUrl) +} + +private class CometNativeExecutorsPartition( + override val index: Int, + val location: String, + val ticket: Array[Byte]) + extends Partition + +/** + * The result partitions of a query that ran on a Comet Native Executor cluster, each read from + * the executor that holds it. + */ +private class CometNativeExecutorsRDD( + sc: SparkContext, + endpoints: Seq[(String, Array[Byte])], + numColumns: Int) + extends RDD[ColumnarBatch](sc, Nil) { + + override protected def getPartitions: Array[Partition] = + endpoints.zipWithIndex.map { case ((location, ticket), index) => + new CometNativeExecutorsPartition(index, location, ticket): Partition + }.toArray + + override def compute(split: Partition, context: TaskContext): Iterator[ColumnarBatch] = { + val partition = split.asInstanceOf[CometNativeExecutorsPartition] + val client = new NativeExecutorsClient() + val nativeUtil = new NativeUtil() + val handle = client.openPartition(partition.location, partition.ticket) + val batches = new Iterator[ColumnarBatch] with AutoCloseable { + private var current: Option[ColumnarBatch] = None + private var pending: Option[ColumnarBatch] = None + private var finished = false + private var closed = false + + override def hasNext: Boolean = { + if (pending.isEmpty && !finished) { + // The consumer has moved past the batch it last got. + current.foreach(_.close()) + current = None + pending = nativeUtil.getNextBatch( + numColumns, + (arrays, schemas) => client.nextBatch(handle, arrays, schemas)) + if (pending.isEmpty) { + finished = true + close() + } + } + pending.isDefined + } + + override def next(): ColumnarBatch = { + if (!hasNext) { + throw new NoSuchElementException("No more batches in this result partition") + } + current = pending + pending = None + current.get + } + + override def close(): Unit = if (!closed) { + closed = true + current.foreach(_.close()) + current = None + pending.foreach(_.close()) + pending = None + client.closePartition(handle) + nativeUtil.close() + } + } + context.addTaskCompletionListener[Unit](_ => batches.close()) + batches + } +} + +/** + * Plans queries for Comet Native Executors: turns the plan Comet would execute into the stage + * graph a cluster runs, and refuses any query that cannot run entirely natively. + */ +object CometNativeExecutors { + + /** Messages of native plans that call back into the JVM, which a cluster doesn't have. */ + private val jvmCallbacks = Set("JvmScalarUdf", "Subquery", "ArrowPythonUdf") + + /** + * Whether Comet runs `plan` on a Comet Native Executor cluster. It must be a whole query: not a + * command, not a stage AQE has split off, and not a plan already replaced. + */ + def applies(conf: SQLConf, plan: SparkPlan, queryStagePrep: Boolean): Boolean = + CometConf.COMET_NATIVE_EXECUTORS_ENABLED.get(conf) && + !(conf.adaptiveExecutionEnabled && !queryStagePrep && plan.isInstanceOf[Exchange]) && + !plan.exists { + case _: CometNativeExecutorsExec | _: AdaptiveSparkPlanExec | _: QueryStageExec | + _: ExecutedCommandExec | _: DataWritingCommandExec | _: V2CommandExec => + true + case _ => false + } + + /** + * Replaces `converted`, the plan Comet would execute for `original`, with a leaf that runs it + * on a Comet Native Executor cluster. An operator on top that only turns the query's result + * into JVM objects, as `Dataset.rdd` adds, stays on the driver above the leaf. + */ + def offload(original: SparkPlan, converted: SparkPlan): SparkPlan = converted match { + case toObjects: DeserializeToObjectExec => + toObjects.withNewChildren(Seq(offload(toObjects.child, toObjects.child))) + case native => + val exec = CometNativeExecutorsExec( + native.output, + buildQuery(native).toByteArray, + CometConf.COMET_NATIVE_EXECUTORS_SCHEDULER_URL.get()) + native.logicalLink.orElse(original.logicalLink).foreach(exec.setLogicalLink) + exec + } + + /** + * The query a Comet Native Executor cluster runs for `plan`, the plan Comet would execute: one + * stage per native block, with an input for each exchange that feeds the block. + * + * @throws CometRuntimeException + * naming the first operator that cannot run on a cluster + */ + def buildQuery(plan: SparkPlan): QueryRequest = { + val builder = new StageGraphBuilder(plan) + val root = plan match { + case ColumnarToRowExec(child) => child + case CometColumnarToRowExec(child) => child + case CometNativeColumnarToRowExec(child) => child + case other => other + } + builder.stageOf(root) + QueryRequest + .newBuilder() + .putAllConf(nativeConf()) + .addAllStages(builder.stages.asJava) + .build() + } + + /** The configs a Spark task would pass to native code with a plan. */ + private def nativeConf(): java.util.Map[String, String] = { + val conf = new java.util.HashMap[String, String]( + ConfigMap.parseFrom(CometExecIterator.serializeCometSQLConfs()).getEntriesMap) + // A Spark task passes the batch size apart from the other configs. + conf.put(CometConf.COMET_BATCH_SIZE.key, CometConf.COMET_BATCH_SIZE.get().toString) + conf + } + + private class StageGraphBuilder(query: SparkPlan) { + val stages: ArrayBuffer[Stage] = ArrayBuffer.empty + private val stageIndexes = new IdentityHashMap[SparkPlan, Integer]() + + /** The index of the stage that runs the native block rooted at `plan`, adding it first. */ + def stageOf(plan: SparkPlan): Int = { + val known = stageIndexes.get(plan) + if (known != null) { + return known + } + val block = plan match { + case native: CometNativeExec if native.serializedPlanOpt.isDefined => native + case other => fail(other, "it does not run natively") + } + val exchanges = inputsOf(block) + val inputs = exchanges.map { exchange => + StageInput + .newBuilder() + .setProducer(stageOf(exchange.child)) + .setPartitioning(partitioningOf(exchange)) + .build() + } + val blockPlan = Operator.parseFrom(block.serializedPlanOpt.plan.get) + findJvmCallback(blockPlan).foreach { name => + fail(block, s"its native plan calls back into the JVM ($name)") + } + + // A block that reads files runs one copy per partition, each naming its own files. + val (common, perPartition) = PlanDataInjector.findAllPlanData(block) + val partitionPlans = if (perPartition.isEmpty) { + Seq.empty + } else { + val base = new PlanDataInjector.CachedPlanData(blockPlan) + (0 until perPartition.values.head.length).map { i => + val partitionData = perPartition.map { case (key, data) => key -> data(i) } + PlanDataInjector.injectPlanData(base, common, partitionData).toByteString + } + } + // The cluster reads it only for a stage that reads no exchange. A stage that does has as + // many partitions as its exchanges deliver. + val numPartitions = + if (partitionPlans.nonEmpty) partitionPlans.length + else block.outputPartitioning.numPartitions + // Declared as Comet's shuffle writer declares its input, since the cluster casts the + // block's output to it as that writer does. + val outputSchema = schema2Proto( + block.output.map(a => StructField(a.name, a.dataType, a.nullable, a.metadata))) + + stages += Stage + .newBuilder() + .setPlan(blockPlan) + .addAllInputs(inputs.asJava) + .addAllPartitionPlans(partitionPlans.asJava) + .setNumPartitions(numPartitions) + .addAllOutputSchema(outputSchema.asJava) + .build() + val index = stages.length - 1 + stageIndexes.put(plan, index) + index + } + + /** + * The exchanges that feed `block`, in the order its native plan's input leaves read them. + */ + private def inputsOf(block: CometNativeExec): Seq[CometShuffleExchangeExec] = { + val inputs = ArrayBuffer.empty[CometShuffleExchangeExec] + def walk(plan: SparkPlan): Unit = plan match { + case exchange: CometShuffleExchangeExec if exchange.shuffleType == CometNativeShuffle => + inputs += exchange + case _: CometNativeScanExec | _: CometIcebergNativeScanExec | _: CometRangeExec => + // Read inside the block. + // Part of the block, even when it has a serialized plan of its own: the conversion + // serializes some operators before their parent joins them into a larger block. + case native: CometNativeExec => + native.children.foreach(walk) + case other => fail(other, "Comet Native Executors do not support it yet") + } + walk(block) + inputs.toSeq + } + + private def partitioningOf( + exchange: CometShuffleExchangeExec): PartitioningOuterClass.Partitioning = { + val partitioning = PartitioningOuterClass.Partitioning.newBuilder() + exchange.outputPartitioning match { + case HashPartitioning(expressions, numPartitions) => + val keys = expressions.map { key => + QueryPlanSerde + .exprToProto(key, exchange.child.output) + .getOrElse(fail(exchange, s"its partitioning key $key cannot run natively")) + } + partitioning.setHashPartition( + PartitioningOuterClass.HashPartition + .newBuilder() + .addAllHashExpression(keys.asJava) + .setNumPartitions(numPartitions)) + case SinglePartition => + partitioning.setSinglePartition(PartitioningOuterClass.SinglePartition.newBuilder()) + case other => + fail(exchange, s"Comet Native Executors do not support $other exchanges yet") + } + partitioning.build() + } + + private def fail(node: SparkPlan, reason: String): Nothing = { + val fallbackReasons = node + .getTagValue(CometExplainInfo.FALLBACK_REASONS) + .getOrElse(Set.empty[String]) + val because = if (fallbackReasons.isEmpty) { + "" + } else { + fallbackReasons.toSeq.sorted.mkString(" Comet could not convert it: ", "; ", ".") + } + throw new CometRuntimeException( + s"Comet Native Executors cannot run this query, because of ${node.nodeName}: " + + s"$reason.$because\n${query.treeString}") + } + } + + /** The first message in `message` that calls back into the JVM, if any. */ + private def findJvmCallback(message: Message): Option[String] = { + val name = message.getDescriptorForType.getName + if (jvmCallbacks.contains(name)) { + Some(name) + } else { + val found = message.getAllFields.asScala.iterator.flatMap { case (field, value) => + if (field.getJavaType != FieldDescriptor.JavaType.MESSAGE) { + Iterator.empty + } else if (field.isRepeated) { + value.asInstanceOf[java.util.List[Message]].asScala.iterator.flatMap(findJvmCallback) + } else { + findJvmCallback(value.asInstanceOf[Message]).iterator + } + } + if (found.hasNext) Some(found.next()) else None + } + } +} diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala new file mode 100644 index 00000000000..bbc5c1476e0 --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.exec + +import java.io.File +import java.net.{ServerSocket, Socket} +import java.nio.file.Files +import java.util.concurrent.TimeUnit + +import scala.util.Try + +import org.apache.spark.SparkConf +import org.apache.spark.sql.{CometTestBase, DataFrame} +import org.apache.spark.sql.comet.CometNativeExecutorsExec +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.spark.sql.functions.{col, count, max, sum} +import org.apache.spark.sql.internal.SQLConf + +import org.apache.comet.CometConf + +/** + * Runs queries on a Comet Native Executor cluster (see + * https://github.com/apache/datafusion-comet/issues/7288): a comet-scheduler and a comet-executor + * started from the binaries in the directory `COMET_NATIVE_EXECUTORS_BIN_DIR` names. The tests + * are skipped when it isn't set. To run them: + * + * {{{ + * cargo build --manifest-path native/Cargo.toml -p datafusion-comet --features native-executors + * CARGO_TARGET_DIR=native/target cargo build --manifest-path native/native-executors/Cargo.toml + * COMET_NATIVE_EXECUTORS_BIN_DIR=native/target/debug ./mvnw test -Dtest=none \ + * -Dsuites=org.apache.comet.exec.CometNativeExecutorsSuite + * }}} + */ +class CometNativeExecutorsSuite extends CometTestBase with AdaptiveSparkPlanHelper { + + private val binDir: Option[File] = + sys.env.get("COMET_NATIVE_EXECUTORS_BIN_DIR").map(new File(_).getAbsoluteFile) + private var processes: Seq[Process] = Nil + private var schedulerUrl: String = _ + + override protected def sparkConf: SparkConf = { + val conf = super.sparkConf + conf.set(CometConf.COMET_SHUFFLE_MODE.key, "native") + conf.set(SQLConf.SHUFFLE_PARTITIONS.key, "4") + conf + } + + override def beforeAll(): Unit = { + super.beforeAll() + binDir.foreach(startCluster) + } + + override def afterAll(): Unit = { + try { + // The executor first, so that it doesn't report to a scheduler that has gone. + processes.reverse.foreach { process => + process.destroy() + if (!process.waitFor(10, TimeUnit.SECONDS)) { + process.destroyForcibly() + } + } + } finally { + super.afterAll() + } + } + + private def startCluster(dir: File): Unit = { + val schedulerPort = freePort() + val flightPort = freePort() + val executorPort = freePort() + val executorGrpcPort = freePort() + val logs = Files.createTempDirectory("comet-native-executors").toFile + logInfo(s"Comet Native Executor cluster logs: $logs") + start( + new File(dir, "comet-scheduler"), + Map( + "COMET_SCHEDULER_BIND_HOST" -> "127.0.0.1", + "COMET_SCHEDULER_PORT" -> schedulerPort.toString, + "COMET_SCHEDULER_FLIGHT_PORT" -> flightPort.toString), + new File(logs, "scheduler.log")) + waitForPort(schedulerPort) + waitForPort(flightPort) + start( + new File(dir, "comet-executor"), + Map( + "COMET_EXECUTOR_BIND_HOST" -> "127.0.0.1", + "COMET_EXECUTOR_PORT" -> executorPort.toString, + "COMET_EXECUTOR_GRPC_PORT" -> executorGrpcPort.toString, + "COMET_SCHEDULER_HOST" -> "127.0.0.1", + "COMET_SCHEDULER_PORT" -> schedulerPort.toString, + "COMET_EXECUTOR_TASKS" -> "4", + "COMET_EXECUTOR_WORK_DIR" -> new File(logs, "work").getPath), + new File(logs, "executor.log")) + waitForPort(executorPort) + schedulerUrl = s"http://127.0.0.1:$flightPort" + } + + private def start(binary: File, env: Map[String, String], log: File): Unit = { + val builder = new ProcessBuilder(binary.getPath) + env.foreach { case (key, value) => builder.environment().put(key, value) } + builder.redirectErrorStream(true).redirectOutput(log) + processes :+= builder.start() + } + + private def freePort(): Int = { + val socket = new ServerSocket(0) + try socket.getLocalPort + finally socket.close() + } + + private def waitForPort(port: Int): Unit = { + val deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(60) + while (Try(new Socket("127.0.0.1", port).close()).isFailure) { + assert(System.nanoTime() < deadline, s"Nothing is listening on port $port") + Thread.sleep(100) + } + } + + private def nativeExecutorsTest(name: String)(body: => Unit): Unit = test(name) { + assume(binDir.isDefined, "COMET_NATIVE_EXECUTORS_BIN_DIR is not set") + body + } + + private def onCluster[T](body: => T): T = withSQLConf( + CometConf.COMET_NATIVE_EXECUTORS_ENABLED.key -> "true", + CometConf.COMET_NATIVE_EXECUTORS_SCHEDULER_URL.key -> schedulerUrl)(body) + + /** Checks that the cluster runs `query` and returns what Spark returns. */ + private def checkOnCluster(query: => DataFrame): Unit = { + val expected = query.collect().toSeq + onCluster { + val df = query + checkAnswer(df, expected) + val plan = df.queryExecution.executedPlan + assert(collect(plan) { case exec: CometNativeExecutorsExec => exec }.size == 1, plan) + } + } + + /** A Parquet table with several files, read as several partitions. */ + private def withTable(body: String => Unit): Unit = withTempPath { dir => + spark + .range(0, 10000, 1, 8) + .selectExpr("id", "id % 7 AS k", "CAST(id AS DOUBLE) * 1.5 AS v") + .write + .parquet(dir.getCanonicalPath) + withSQLConf(SQLConf.FILES_MAX_PARTITION_BYTES.key -> "16384") { + body(dir.getCanonicalPath) + } + } + + nativeExecutorsTest("scan, filter and project") { + withTable { path => + checkOnCluster(spark.read.parquet(path).where("k > 2").selectExpr("id", "v * 2 AS w")) + } + } + + nativeExecutorsTest("aggregate across a hash shuffle") { + withTable { path => + checkOnCluster(spark.read.parquet(path).groupBy("k").agg(sum("v"), count("*"))) + } + } + + nativeExecutorsTest("aggregate across a hash shuffle without AQE") { + withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") { + withTable { path => + checkOnCluster(spark.read.parquet(path).groupBy("k").agg(sum("v"), max("id"))) + } + } + } + + nativeExecutorsTest("global aggregate across a single-partition shuffle") { + withTable { path => + checkOnCluster(spark.read.parquet(path).agg(sum("id"), max("v"), count("*"))) + } + } + + Seq(true, false).foreach { sortMerge => + val join = if (sortMerge) "sort-merge" else "hash" + nativeExecutorsTest(s"shuffled $join join") { + withSQLConf( + SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + SQLConf.PREFER_SORTMERGEJOIN.key -> sortMerge.toString) { + withTable { path => + val left = spark.read.parquet(path).select("id", "k") + val right = spark.read + .parquet(path) + .where("id < 10") + .select(col("k").as("k2"), col("v")) + checkOnCluster(left.join(right, left("k") === right("k2")).select("id", "v")) + } + } + } + } + + nativeExecutorsTest("a query the cluster cannot run fails, naming the operator") { + withTable { path => + onCluster { + // A global sort needs a range exchange, which the prototype doesn't support. + val error = intercept[Exception](spark.read.parquet(path).orderBy("v").collect()) + val messages = Iterator + .iterate[Throwable](error)(_.getCause) + .takeWhile(_ != null) + .map(e => String.valueOf(e.getMessage)) + .toSeq + assert( + messages.exists(message => + message.contains("Comet Native Executors cannot run this query") && + message.contains("rangepartitioning")), + error) + } + } + } +} From d18c549fa281596e715fc18ade66dbc09c284b31 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 28 Oct 2026 09:53:31 -0600 Subject: [PATCH 07/36] feat: run sorts, and with them TPC-H q1, on Comet Native Executors The driver now serializes a range exchange as a RangePartition with its sort keys and no bounds. Until Ballista can plan a range shuffle for a physical plan (apache/datafusion-ballista#2680), the scheduler reads a range exchange as one partition, so the stage that sorts it runs in a single task. That is correct for a global sort, if serial. CometNativeExecutorsSuite runs TPC-H q1, with and without its order by, on the lineitem table under SPARK_TPCH_DATA when it is set, and on generated data with the same schema otherwise. The query that the cluster must refuse is now a broadcast join. See #7288. --- .../sql/comet/CometNativeExecutorsExec.scala | 14 +++- .../exec/CometNativeExecutorsSuite.scala | 73 ++++++++++++++++++- 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala index ef9c4de17c5..1125fc45d0c 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala @@ -28,7 +28,7 @@ import org.apache.spark.{Partition, SparkContext, TaskContext} import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, SinglePartition} +import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, RangePartitioning, SinglePartition} import org.apache.spark.sql.comet.execution.shuffle.{CometNativeShuffle, CometShuffleExchangeExec} import org.apache.spark.sql.execution.{ColumnarToRowExec, DeserializeToObjectExec, LeafExecNode, SparkPlan} import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, QueryStageExec} @@ -325,6 +325,18 @@ object CometNativeExecutors { .newBuilder() .addAllHashExpression(keys.asJava) .setNumPartitions(numPartitions)) + case RangePartitioning(ordering, numPartitions) => + // Without bounds: the cluster decides how to range-partition. + val sortOrders = ordering.map { order => + QueryPlanSerde + .exprToProto(order, exchange.child.output) + .getOrElse(fail(exchange, s"its sort key $order cannot run natively")) + } + partitioning.setRangePartition( + PartitioningOuterClass.RangePartition + .newBuilder() + .addAllSortOrders(sortOrders.asJava) + .setNumPartitions(numPartitions)) case SinglePartition => partitioning.setSinglePartition(PartitioningOuterClass.SinglePartition.newBuilder()) case other => diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala index bbc5c1476e0..0ae64710c26 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala @@ -210,11 +210,78 @@ class CometNativeExecutorsSuite extends CometTestBase with AdaptiveSparkPlanHelp } } + /** + * TPC-H's `lineitem`: the table under `SPARK_TPCH_DATA` when it is set, as for + * `CometTPCHQuerySuite`, and otherwise generated data with the same schema. + */ + private def withLineitem(body: => Unit): Unit = sys.env.get("SPARK_TPCH_DATA") match { + case Some(path) => + spark.read.parquet(s"$path/lineitem").createOrReplaceTempView("lineitem") + withSQLConf(SQLConf.FILES_MAX_PARTITION_BYTES.key -> "32m")(body) + case None => + withTempPath { dir => + spark + .range(0, 100000, 1, 8) + .selectExpr( + "CAST(id % 50 + 1 AS DECIMAL(15, 2)) AS l_quantity", + "CAST((id % 1000) * 1.37 + 900 AS DECIMAL(15, 2)) AS l_extendedprice", + "CAST((id % 11) / 100.0 AS DECIMAL(15, 2)) AS l_discount", + "CAST((id % 9) / 100.0 AS DECIMAL(15, 2)) AS l_tax", + "ELT(CAST(id % 3 AS INT) + 1, 'A', 'N', 'R') AS l_returnflag", + "ELT(CAST(id % 2 AS INT) + 1, 'F', 'O') AS l_linestatus", + "DATE_ADD(DATE '1992-01-01', CAST(id % 2500 AS INT)) AS l_shipdate") + .write + .parquet(dir.getCanonicalPath) + spark.read.parquet(dir.getCanonicalPath).createOrReplaceTempView("lineitem") + withSQLConf(SQLConf.FILES_MAX_PARTITION_BYTES.key -> "131072")(body) + } + } + + private val q1Aggregate = """ + |select + | l_returnflag, + | l_linestatus, + | sum(l_quantity) as sum_qty, + | sum(l_extendedprice) as sum_base_price, + | sum(l_extendedprice * (1 - l_discount)) as sum_disc_price, + | sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge, + | avg(l_quantity) as avg_qty, + | avg(l_extendedprice) as avg_price, + | avg(l_discount) as avg_disc, + | count(*) as count_order + |from + | lineitem + |where + | l_shipdate <= date '1998-12-01' - interval '90' day + |group by + | l_returnflag, + | l_linestatus""".stripMargin + + nativeExecutorsTest("TPC-H q1 without its order by") { + withLineitem { + checkOnCluster(sql(q1Aggregate)) + } + } + + nativeExecutorsTest("TPC-H q1") { + withLineitem { + // A new Dataset each time, since a Dataset keeps the plan it was first run with. + def q1 = sql(s"$q1Aggregate\norder by\n l_returnflag,\n l_linestatus") + checkOnCluster(q1) + // checkAnswer ignores the order of the rows. + val expected = q1.collect().toSeq + onCluster(assert(q1.collect().toSeq == expected)) + } + } + nativeExecutorsTest("a query the cluster cannot run fails, naming the operator") { withTable { path => onCluster { - // A global sort needs a range exchange, which the prototype doesn't support. - val error = intercept[Exception](spark.read.parquet(path).orderBy("v").collect()) + // A small join is broadcast, and the prototype doesn't support broadcast exchanges. + val left = spark.read.parquet(path) + val right = spark.read.parquet(path).where("id < 10").select(col("k").as("k2")) + val error = + intercept[Exception](left.join(right, left("k") === right("k2")).collect()) val messages = Iterator .iterate[Throwable](error)(_.getCause) .takeWhile(_ != null) @@ -223,7 +290,7 @@ class CometNativeExecutorsSuite extends CometTestBase with AdaptiveSparkPlanHelp assert( messages.exists(message => message.contains("Comet Native Executors cannot run this query") && - message.contains("rangepartitioning")), + message.contains("CometBroadcastExchange")), error) } } From 21069b580aaf307d22f567a3f5b54515bf30dc16 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 28 Oct 2026 08:06:34 -0600 Subject: [PATCH 08/36] ci: list CometNativeExecutorsSuite as a manual suite It runs only against comet-scheduler and comet-executor binaries named by COMET_NATIVE_EXECUTORS_BIN_DIR, so the PR build matrix would only skip it. --- dev/ci/check-suites.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/ci/check-suites.py b/dev/ci/check-suites.py index 68bf972c5cf..5ced2ba6971 100644 --- a/dev/ci/check-suites.py +++ b/dev/ci/check-suites.py @@ -42,6 +42,7 @@ def file_to_class_name(path: Path) -> str | None: "org.apache.comet.CometIcebergSchedulerFailureSuite", # manual, multiple hosts + SIGKILL "org.apache.comet.cloud.s3.CometS3CredentialBridgeSuite", # manual test suite "org.apache.comet.cloud.s3.HadoopS3ACredentialProviderAdapterBridgeSuite", # manual test suite + "org.apache.comet.exec.CometNativeExecutorsSuite", # manual, needs comet-scheduler and comet-executor binaries "org.apache.comet.shuffle.CelebornReflectionCompatibilitySuite", # dedicated version matrix "org.apache.spark.sql.comet.CometPlanStabilitySuite", # abstract "org.apache.spark.sql.comet.ParquetDatetimeRebaseSuite", # abstract From 9cf6478472ada56cde4c1b66db6af87afa624798 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 31 Oct 2026 06:32:58 -0600 Subject: [PATCH 09/36] fix: compile CometNativeExecutorsSuite on Spark 3.x and under strict warnings withSQLConf returns Unit on Spark 3.x, so onCluster takes and returns Unit. NativeExecutorsClient's JNI declarations join Native's under the strict-warnings unused-params filter. --- pom.xml | 4 ++-- .../org/apache/comet/exec/CometNativeExecutorsSuite.scala | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index a215e2a818f..e510c08eb66 100644 --- a/pom.xml +++ b/pom.xml @@ -1010,7 +1010,7 @@ under the License. -Ywarn-value-discard -Ywarn-unused:imports,patvars,privates,locals,params,-implicits - -Wconf:cat=unused-params&site=org\.apache\.comet\.Native\..*:s + -Wconf:cat=unused-params&site=org\.apache\.comet\.Native(ExecutorsClient)?\..*:s -Wconf:cat=unused-params&src=.*/src/[a-z]+/spark-[^/]+/.*:s -Wconf:cat=unused-params&site=org\.apache\.comet\.rules\.CometScanContrib\.tryTransformV[12]\..*:s @@ -1055,7 +1055,7 @@ under the License. -Ywarn-numeric-widen -Ywarn-unused:imports,patvars,privates,locals,params,-implicits - -Wconf:cat=unused-params&site=org\.apache\.comet\.Native\..*:s + -Wconf:cat=unused-params&site=org\.apache\.comet\.Native(ExecutorsClient)?\..*:s -Wconf:cat=unused-params&src=.*/src/[a-z]+/spark-[^/]+/.*:s -Wconf:cat=unused-params&site=org\.apache\.comet\.rules\.CometScanContrib\.tryTransformV[12]\..*:s diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala index 0ae64710c26..468061d76a1 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala @@ -138,7 +138,7 @@ class CometNativeExecutorsSuite extends CometTestBase with AdaptiveSparkPlanHelp body } - private def onCluster[T](body: => T): T = withSQLConf( + private def onCluster(body: => Unit): Unit = withSQLConf( CometConf.COMET_NATIVE_EXECUTORS_ENABLED.key -> "true", CometConf.COMET_NATIVE_EXECUTORS_SCHEDULER_URL.key -> schedulerUrl)(body) From cea5fc8a47aa243243ae21a02e3c89684eba8b78 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 31 Oct 2026 11:13:07 -0600 Subject: [PATCH 10/36] build: build Comet Native Executors on Ballista's comet-native branch The branch is Ballista's main branch with the enhancements Comet Native Executors need that haven't merged yet: apache/datafusion-ballista#2682, #2721, #2722 and #2723. Cargo.lock pins its commit. --- native/native-executors/Cargo.lock | 19 ++++++++----------- native/native-executors/Cargo.toml | 12 +++++++----- native/native-executors/README.md | 14 ++++++++++---- native/native-executors/src/assembly.rs | 7 ++++--- 4 files changed, 29 insertions(+), 23 deletions(-) diff --git a/native/native-executors/Cargo.lock b/native/native-executors/Cargo.lock index 4d21ac8a3e5..0a52da7f50d 100644 --- a/native/native-executors/Cargo.lock +++ b/native/native-executors/Cargo.lock @@ -999,8 +999,7 @@ dependencies = [ [[package]] name = "ballista-core" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a090511fa9b3661dbf17444ecbbf5efcd11793e140ddd7fe2fb85f2de88f561" +source = "git+https://github.com/apache/datafusion-ballista?branch=comet-native#2efa802ffd6f454211d516b660a3f9db648bab65" dependencies = [ "arrow-flight", "async-trait", @@ -1029,8 +1028,7 @@ dependencies = [ [[package]] name = "ballista-executor" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "403cdac6a4e822e6025348f4f7e43765ba0ca176ca1f225888a9eeb90b9c66ce" +source = "git+https://github.com/apache/datafusion-ballista?branch=comet-native#2efa802ffd6f454211d516b660a3f9db648bab65" dependencies = [ "arrow", "arrow-flight", @@ -1059,8 +1057,7 @@ dependencies = [ [[package]] name = "ballista-scheduler" version = "55.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "845bb1c2c763ae2b50ecfd9debdfe8e092d13a4fefe84c20e9262bb191a1f1e7" +source = "git+https://github.com/apache/datafusion-ballista?branch=comet-native#2efa802ffd6f454211d516b660a3f9db648bab65" dependencies = [ "arrow-flight", "async-trait", @@ -3471,7 +3468,7 @@ dependencies = [ [[package]] name = "iceberg" version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust?rev=1f3bc34058e3f3e45336bc5428f37b92649a608b#1f3bc34058e3f3e45336bc5428f37b92649a608b" +source = "git+https://github.com/apache/iceberg-rust?rev=b154603a710d730d2f43f8af4db257cc137237ef#b154603a710d730d2f43f8af4db257cc137237ef" dependencies = [ "aes-gcm", "anyhow", @@ -3529,7 +3526,7 @@ dependencies = [ [[package]] name = "iceberg-property-macro" version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust?rev=1f3bc34058e3f3e45336bc5428f37b92649a608b#1f3bc34058e3f3e45336bc5428f37b92649a608b" +source = "git+https://github.com/apache/iceberg-rust?rev=b154603a710d730d2f43f8af4db257cc137237ef#b154603a710d730d2f43f8af4db257cc137237ef" dependencies = [ "proc-macro2", "quote", @@ -3539,7 +3536,7 @@ dependencies = [ [[package]] name = "iceberg-storage-opendal" version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust?rev=1f3bc34058e3f3e45336bc5428f37b92649a608b#1f3bc34058e3f3e45336bc5428f37b92649a608b" +source = "git+https://github.com/apache/iceberg-rust?rev=b154603a710d730d2f43f8af4db257cc137237ef#b154603a710d730d2f43f8af4db257cc137237ef" dependencies = [ "anyhow", "async-trait", @@ -4081,9 +4078,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.18.5" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef9ac18847474e638e3702b76c65d4eb93428471a74778ef0f1be711717f89b5" +checksum = "4a9f71e3591c7ec44d3e563e87bf14ddc2bada8519dca046e5b0e560cd004e2d" dependencies = [ "hashbrown 0.17.1", ] diff --git a/native/native-executors/Cargo.toml b/native/native-executors/Cargo.toml index 117cd7e3df4..f83d7536256 100644 --- a/native/native-executors/Cargo.toml +++ b/native/native-executors/Cargo.toml @@ -19,8 +19,10 @@ # DataFusion Ballista. See https://github.com/apache/datafusion-comet/issues/7288. # # This is its own workspace, with its own lockfile, so that the main Comet build never depends -# on Ballista. It builds Comet's native crates by path together with the Ballista release on the -# same DataFusion major version. +# on Ballista. It builds Comet's native crates by path together with Ballista's `comet-native` +# branch, which is Ballista's main branch with the enhancements Comet Native Executors need that +# haven't merged yet (apache/datafusion-ballista#2682, #2721, #2722 and #2723), on the same +# DataFusion major version. [workspace] [package] @@ -35,9 +37,9 @@ publish = false [dependencies] arrow-flight = "59.2.0" -ballista-core = "55.0.0" -ballista-executor = { version = "55.0.0", default-features = false, features = ["arrow-ipc-optimizations"] } -ballista-scheduler = { version = "55.0.0", default-features = false } +ballista-core = { git = "https://github.com/apache/datafusion-ballista", branch = "comet-native" } +ballista-executor = { git = "https://github.com/apache/datafusion-ballista", branch = "comet-native", default-features = false, features = ["arrow-ipc-optimizations"] } +ballista-scheduler = { git = "https://github.com/apache/datafusion-ballista", branch = "comet-native", default-features = false } datafusion = { version = "55.2.0", default-features = false } # Comet's core crate, under the name of its library. Without the default hdfs-opendal feature, # which links libjvm through libhdfs. diff --git a/native/native-executors/README.md b/native/native-executors/README.md index 4224ab1de7a..b332496b031 100644 --- a/native/native-executors/README.md +++ b/native/native-executors/README.md @@ -81,7 +81,13 @@ binaries and runs queries through the scheduler's Flight service, as a driver wo ## Keeping it building -This workspace builds Comet's native crates by path against the Ballista release on the same -DataFusion major version, and CI builds and tests it in the `linux-test-native-executors` job. When -Comet moves to a DataFusion major that Ballista hasn't released yet, the upgrade can turn that job -off until Ballista catches up. +This workspace builds Comet's native crates by path against Ballista's +[`comet-native`](https://github.com/apache/datafusion-ballista/tree/comet-native) branch: Ballista's +main branch with the enhancements Comet Native Executors need that haven't merged yet +([#2682](https://github.com/apache/datafusion-ballista/pull/2682), +[#2721](https://github.com/apache/datafusion-ballista/pull/2721), +[#2722](https://github.com/apache/datafusion-ballista/pull/2722) and +[#2723](https://github.com/apache/datafusion-ballista/pull/2723)), on the same DataFusion major +version. `Cargo.lock` pins the branch's commit. CI builds and tests the workspace in the +`linux-test-native-executors` job. When Comet moves to a DataFusion major that Ballista hasn't +caught up with, the upgrade can turn that job off until it does. diff --git a/native/native-executors/src/assembly.rs b/native/native-executors/src/assembly.rs index 8464d97fbc9..f8ad3e3af0b 100644 --- a/native/native-executors/src/assembly.rs +++ b/native/native-executors/src/assembly.rs @@ -179,9 +179,10 @@ fn exchange( Partitioning::Hash(keys, hash.num_partitions as usize), )?)) } - // Ballista can't plan a range shuffle for a physical plan yet - // (https://github.com/apache/datafusion-ballista/issues/2680), so the consuming stage - // reads a range exchange as one partition too. That sorts correctly, in a single task. + // Ballista plans a range shuffle only for a range partitioning with split points + // (https://github.com/apache/datafusion-ballista/pull/2722), and the driver doesn't + // sample its input for them yet, so the consuming stage reads a range exchange as one + // partition too. That sorts correctly, in a single task. Some(PartitioningStruct::SinglePartition(_) | PartitioningStruct::RangePartition(_)) => { Ok(Arc::new(CoalescePartitionsExec::new(producer))) } From 4c24b2ab089927f2bb87b17ef32bcf576e80776a Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 31 Oct 2026 11:13:07 -0600 Subject: [PATCH 11/36] fix: let an optimizer rebuild CometFragmentExec around an exchange it moves Ballista runs EnsureRequirements on each stage, which takes away the coalesce above a stage's input and puts one back only where the parent requires a single partition. It rebuilds the node in between, so CometFragmentExec now checks that its inputs line up with its partitions when it runs rather than when it is built, and a stage with one partition requires each input as one partition. --- .../execution/native_executors/fragment.rs | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/native/core/src/execution/native_executors/fragment.rs b/native/core/src/execution/native_executors/fragment.rs index 4e4aec790e0..34d955a930d 100644 --- a/native/core/src/execution/native_executors/fragment.rs +++ b/native/core/src/execution/native_executors/fragment.rs @@ -126,15 +126,6 @@ impl CometFragmentExec { )); } let partitions = descriptor.output_partitioning().partition_count(); - for (index, (input, broadcast)) in inputs.iter().zip(&fragment.broadcast).enumerate() { - let expected = if *broadcast { 1 } else { partitions }; - let actual = input.output_partitioning().partition_count(); - if actual != expected { - return Err(internal_datafusion_err!( - "CometFragmentExec input {index} has {actual} partitions, expected {expected}" - )); - } - } let props = Arc::new(PlanProperties::new( EquivalenceProperties::new(fragment_schema(&fragment.output_schema)), Partitioning::UnknownPartitioning(partitions), @@ -151,6 +142,27 @@ impl CometFragmentExec { pub fn fragment(&self) -> &Arc { &self.fragment } + + /// Checks that each input's partitions line up with the descriptor's. This is checked when the + /// plan runs rather than when it is built, since an optimizer can rebuild the node with an + /// input whose exchange it has taken away and is about to put back. + fn check_inputs(&self) -> Result<()> { + let partitions = self.props.output_partitioning().partition_count(); + for (index, (input, broadcast)) in self.children[1..] + .iter() + .zip(&self.fragment.broadcast) + .enumerate() + { + let expected = if *broadcast { 1 } else { partitions }; + let actual = input.output_partitioning().partition_count(); + if actual != expected { + return Err(internal_datafusion_err!( + "CometFragmentExec input {index} has {actual} partitions, expected {expected}" + )); + } + } + Ok(()) + } } /// The schema of a fragment's output: the types Spark declared for it, every column nullable. @@ -193,10 +205,13 @@ impl ExecutionPlan for CometFragmentExec { /// The descriptor's and the partitioned inputs' partitions line up with this plan's, while a /// broadcast input is read whole by every partition, so a distributed engine must not split - /// it between tasks. + /// it between tasks. A stage with one partition reads every input as one partition, which + /// keeps an optimizer such as `EnsureRequirements` from removing the coalesce that gathers + /// it. fn input_distribution_requirements(&self) -> InputDistributionRequirements { + let single = self.props.output_partitioning().partition_count() == 1; let inputs = self.fragment.broadcast.iter().map(|broadcast| { - if *broadcast { + if *broadcast || single { Distribution::SinglePartition } else { Distribution::UnspecifiedDistribution @@ -237,6 +252,7 @@ impl ExecutionPlan for CometFragmentExec { partition: usize, context: Arc, ) -> Result { + self.check_inputs()?; let descriptor = Arc::clone(&self.children[0]); let inputs = self.children[1..] .iter() From fc3a5890a7bda4e0599c850eff6eee9707cb4a10 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 31 Oct 2026 11:13:07 -0600 Subject: [PATCH 12/36] feat: run broadcast joins, scalar subqueries and top-K on Comet Native Executors The stage graph builder now takes a broadcast exchange as a broadcast input and resolves a reused exchange to the stage it reuses. A query's scalar subqueries are expressions of CometNativeExecutorsExec, so Spark runs them before the query, and their results replace them in the plan as literals before it is submitted. CometTakeOrderedAndProjectExec becomes a stage that reads its child as one partition and runs Comet's final top-K plan. --- .../sql/comet/CometNativeExecutorsExec.scala | 187 +++++++++++++++--- 1 file changed, 162 insertions(+), 25 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala index 1125fc45d0c..e3e602302f9 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala @@ -27,23 +27,24 @@ import scala.jdk.CollectionConverters._ import org.apache.spark.{Partition, SparkContext, TaskContext} import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, Literal} import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, RangePartitioning, SinglePartition} import org.apache.spark.sql.comet.execution.shuffle.{CometNativeShuffle, CometShuffleExchangeExec} -import org.apache.spark.sql.execution.{ColumnarToRowExec, DeserializeToObjectExec, LeafExecNode, SparkPlan} +import org.apache.spark.sql.execution.{ColumnarToRowExec, DeserializeToObjectExec, LeafExecNode, ScalarSubquery, SparkPlan} import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, QueryStageExec} import org.apache.spark.sql.execution.command.{DataWritingCommandExec, ExecutedCommandExec} import org.apache.spark.sql.execution.datasources.v2.V2CommandExec -import org.apache.spark.sql.execution.exchange.Exchange +import org.apache.spark.sql.execution.exchange.{Exchange, ReusedExchangeExec} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructField import org.apache.spark.sql.vectorized.ColumnarBatch +import com.google.protobuf.ByteString import com.google.protobuf.Descriptors.FieldDescriptor import com.google.protobuf.Message import org.apache.comet.{CometConf, CometExecIterator, CometExplainInfo, CometRuntimeException, NativeExecutorsClient} -import org.apache.comet.serde.{PartitioningOuterClass, QueryPlanSerde} +import org.apache.comet.serde.{ExprOuterClass, PartitioningOuterClass, QueryPlanSerde} import org.apache.comet.serde.Config.ConfigMap import org.apache.comet.serde.NativeExecutors.{QueryRequest, ResultEndpoints, Stage, StageInput} import org.apache.comet.serde.OperatorOuterClass.Operator @@ -60,11 +61,15 @@ import org.apache.comet.vector.NativeUtil * * @param query * the serialized `QueryRequest` built by [[CometNativeExecutors.buildQuery]] + * @param subqueryResults + * the query's scalar subqueries. They are expressions of this node, so Spark runs them before + * it executes the node, and their results replace them in the query before it is submitted. */ case class CometNativeExecutorsExec( override val output: Seq[Attribute], query: Array[Byte], - schedulerUrl: String) + schedulerUrl: String, + subqueryResults: Seq[Expression] = Nil) extends LeafExecNode { override def supportsColumnar: Boolean = true @@ -73,8 +78,11 @@ case class CometNativeExecutorsExec( throw new UnsupportedOperationException(s"$nodeName only produces columnar output") override protected def doExecuteColumnar(): RDD[ColumnarBatch] = { + val request = CometNativeExecutors.withSubqueryResults( + query, + subqueryResults.collect { case sub: ScalarSubquery => sub }) val endpoints = - ResultEndpoints.parseFrom(new NativeExecutorsClient().submit(schedulerUrl, query)) + ResultEndpoints.parseFrom(new NativeExecutorsClient().submit(schedulerUrl, request)) val partitions = endpoints.getEndpointsList.asScala.map { endpoint => (endpoint.getLocation, endpoint.getTicket.toByteArray) } @@ -163,8 +171,12 @@ private class CometNativeExecutorsRDD( */ object CometNativeExecutors { - /** Messages of native plans that call back into the JVM, which a cluster doesn't have. */ - private val jvmCallbacks = Set("JvmScalarUdf", "Subquery", "ArrowPythonUdf") + /** + * Messages of native plans that call back into the JVM, which a cluster doesn't have. A scalar + * subquery's `Subquery` is not one of them: the driver runs the subquery and puts its result in + * the plan before submitting it. + */ + private val jvmCallbacks = Set("JvmScalarUdf", "ArrowPythonUdf") /** * Whether Comet runs `plan` on a Comet Native Executor cluster. It must be a whole query: not a @@ -192,7 +204,8 @@ object CometNativeExecutors { val exec = CometNativeExecutorsExec( native.output, buildQuery(native).toByteArray, - CometConf.COMET_NATIVE_EXECUTORS_SCHEDULER_URL.get()) + CometConf.COMET_NATIVE_EXECUTORS_SCHEDULER_URL.get(), + scalarSubqueries(native)) native.logicalLink.orElse(original.logicalLink).foreach(exec.setLogicalLink) exec } @@ -220,6 +233,89 @@ object CometNativeExecutors { .build() } + /** The scalar subqueries of `plan`'s expressions, through exchanges, each once. */ + private def scalarSubqueries(plan: SparkPlan): Seq[Expression] = { + val found = ArrayBuffer.empty[ScalarSubquery] + def walk(node: SparkPlan): Unit = { + node.expressions.foreach(_.foreach { + case sub: ScalarSubquery if !found.exists(_.exprId == sub.exprId) => found += sub + case _ => + }) + node match { + case reused: ReusedExchangeExec => walk(reused.child) + case _ => node.children.foreach(walk) + } + } + walk(plan) + found.toSeq + } + + /** + * `query` with each scalar subquery's `Subquery` expression replaced by a literal of its + * result. The subqueries must have run. + */ + def withSubqueryResults(query: Array[Byte], subqueries: Seq[ScalarSubquery]): Array[Byte] = { + if (subqueries.isEmpty) { + return query + } + val literals: Map[Long, ExprOuterClass.Expr] = subqueries.map { sub => + val literal = Literal.create(sub.eval(null), sub.dataType) + val expr = QueryPlanSerde + .exprToProto(literal, Seq.empty) + .getOrElse(throw new CometRuntimeException( + s"Comet Native Executors cannot pass the result of $sub to the cluster")) + sub.exprId.id -> expr + }.toMap + val request = QueryRequest.parseFrom(query).toBuilder + (0 until request.getStagesCount).foreach { i => + val stage = request.getStages(i).toBuilder + stage.setPlan(replaceSubqueries(stage.getPlan, literals).asInstanceOf[Operator]) + (0 until stage.getPartitionPlansCount).foreach { p => + val plan = Operator.parseFrom(stage.getPartitionPlans(p)) + stage.setPartitionPlans( + p, + ByteString.copyFrom(replaceSubqueries(plan, literals).toByteArray)) + } + request.setStages(i, stage) + } + request.build().toByteArray + } + + private def replaceSubqueries( + message: Message, + literals: Map[Long, ExprOuterClass.Expr]): Message = message match { + case expr: ExprOuterClass.Expr if expr.hasSubquery => + literals.getOrElse( + expr.getSubquery.getId, + throw new CometRuntimeException( + s"Comet Native Executors found no result for subquery ${expr.getSubquery.getId}")) + case _ => + val builder = message.toBuilder + var changed = false + message.getAllFields.asScala.foreach { case (field, value) => + if (field.getJavaType == FieldDescriptor.JavaType.MESSAGE) { + if (field.isRepeated) { + value.asInstanceOf[java.util.List[Message]].asScala.zipWithIndex.foreach { + case (element, index) => + val replaced = replaceSubqueries(element, literals) + if (replaced ne element) { + builder.setRepeatedField(field, index, replaced) + changed = true + } + } + } else { + val element = value.asInstanceOf[Message] + val replaced = replaceSubqueries(element, literals) + if (replaced ne element) { + builder.setField(field, replaced) + changed = true + } + } + } + } + if (changed) builder.build() else message + } + /** The configs a Spark task would pass to native code with a plan. */ private def nativeConf(): java.util.Map[String, String] = { val conf = new java.util.HashMap[String, String]( @@ -240,21 +336,25 @@ object CometNativeExecutors { return known } val block = plan match { + case take: CometTakeOrderedAndProjectExec => return takeOrderedStage(take) case native: CometNativeExec if native.serializedPlanOpt.isDefined => native case other => fail(other, "it does not run natively") } - val exchanges = inputsOf(block) - val inputs = exchanges.map { exchange => - StageInput - .newBuilder() - .setProducer(stageOf(exchange.child)) - .setPartitioning(partitioningOf(exchange)) - .build() + val inputs = inputsOf(block).map { + case exchange: CometShuffleExchangeExec => + StageInput + .newBuilder() + .setProducer(stageOf(exchange.child)) + .setPartitioning(partitioningOf(exchange)) + .build() + case broadcast => + StageInput + .newBuilder() + .setProducer(stageOf(broadcast.children.head)) + .setBroadcast(true) + .build() } val blockPlan = Operator.parseFrom(block.serializedPlanOpt.plan.get) - findJvmCallback(blockPlan).foreach { name => - fail(block, s"its native plan calls back into the JVM ($name)") - } // A block that reads files runs one copy per partition, each naming its own files. val (common, perPartition) = PlanDataInjector.findAllPlanData(block) @@ -272,11 +372,43 @@ object CometNativeExecutors { val numPartitions = if (partitionPlans.nonEmpty) partitionPlans.length else block.outputPartitioning.numPartitions + addStage(plan, blockPlan, inputs, partitionPlans, numPartitions, block.output) + } + + /** + * A stage for `take` that reads its child's whole output as one partition and keeps the first + * rows in order, as Comet's final top-K plan does after its shuffle to one partition. + */ + private def takeOrderedStage(take: CometTakeOrderedAndProjectExec): Int = { + val input = StageInput + .newBuilder() + .setProducer(stageOf(take.child)) + .setPartitioning( + PartitioningOuterClass.Partitioning + .newBuilder() + .setSinglePartition(PartitioningOuterClass.SinglePartition.newBuilder())) + .build() + // More than one input partition, so that the plan sorts across all of them. + val blockPlan = take + .finalNativePlan(inputPartitions = 2) + .getOrElse(fail(take, "its top-K plan cannot run natively")) + addStage(take, blockPlan, Seq(input), Seq.empty, 1, take.output) + } + + private def addStage( + plan: SparkPlan, + blockPlan: Operator, + inputs: Seq[StageInput], + partitionPlans: Seq[ByteString], + numPartitions: Int, + output: Seq[Attribute]): Int = { + findJvmCallback(blockPlan).foreach { name => + fail(plan, s"its native plan calls back into the JVM ($name)") + } // Declared as Comet's shuffle writer declares its input, since the cluster casts the // block's output to it as that writer does. - val outputSchema = schema2Proto( - block.output.map(a => StructField(a.name, a.dataType, a.nullable, a.metadata))) - + val outputSchema = + schema2Proto(output.map(a => StructField(a.name, a.dataType, a.nullable, a.metadata))) stages += Stage .newBuilder() .setPlan(blockPlan) @@ -291,13 +423,18 @@ object CometNativeExecutors { } /** - * The exchanges that feed `block`, in the order its native plan's input leaves read them. + * The exchanges that feed `block`, in the order its native plan's input leaves read them: + * native shuffles and broadcasts, with a reused exchange resolved to the one it reuses. */ - private def inputsOf(block: CometNativeExec): Seq[CometShuffleExchangeExec] = { - val inputs = ArrayBuffer.empty[CometShuffleExchangeExec] + private def inputsOf(block: CometNativeExec): Seq[SparkPlan] = { + val inputs = ArrayBuffer.empty[SparkPlan] def walk(plan: SparkPlan): Unit = plan match { + case reused: ReusedExchangeExec => + walk(reused.child) case exchange: CometShuffleExchangeExec if exchange.shuffleType == CometNativeShuffle => inputs += exchange + case broadcast: CometBroadcastExchangeExec => + inputs += broadcast case _: CometNativeScanExec | _: CometIcebergNativeScanExec | _: CometRangeExec => // Read inside the block. // Part of the block, even when it has a serialized plan of its own: the conversion From f64f8e6cdbed44e685474f8b0901b01eda168591 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 31 Oct 2026 11:13:07 -0600 Subject: [PATCH 13/36] test: run the 22 TPC-H queries on Comet Native Executors With SPARK_TPCH_DATA set, CometNativeExecutorsSuite runs each TPC-H query on the cluster and compares its rows, in order when the query sorts, with Spark's. TPCH_QUERIES picks a subset. The suite also covers a broadcast join and a scalar subquery, and the query the cluster must refuse is now a sort that stays in Spark. --- .../exec/CometNativeExecutorsSuite.scala | 108 +++++++++++++++--- 1 file changed, 92 insertions(+), 16 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala index 468061d76a1..ef3a585fd1f 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala @@ -274,24 +274,100 @@ class CometNativeExecutorsSuite extends CometTestBase with AdaptiveSparkPlanHelp } } + private val tpchTables = + Seq("customer", "lineitem", "nation", "orders", "part", "partsupp", "region", "supplier") + + /** + * The 22 TPC-H queries, on the tables under `SPARK_TPCH_DATA`. `TPCH_QUERIES`, a + * comma-separated list of query numbers, runs only those. + */ + private val tpchQueries = { + val only = sys.env.get("TPCH_QUERIES").map(_.split(",").map(_.trim.toInt).toSet) + (1 to 22).filter(q => only.forall(_.contains(q))) + } + + tpchQueries.foreach { q => + nativeExecutorsTest(s"TPC-H query $q") { + val data = sys.env.get("SPARK_TPCH_DATA") + assume(data.isDefined, "SPARK_TPCH_DATA is not set") + tpchTables.foreach { table => + spark.read.parquet(s"${data.get}/$table").createOrReplaceTempView(table) + } + val file = + new File(getClass.getResource("/").toURI.resolve("../../../benchmarks/tpc/queries/tpch/")) + val text = new String(Files.readAllBytes(new File(file, s"q$q.sql").toPath), "UTF-8") + val statements = text + .split("\n") + .filterNot(_.trim.startsWith("--")) + .mkString("\n") + .split(";") + .map(_.trim) + .filter(_.nonEmpty) + // q15 creates a view, queries it and drops it. + val (setup, rest) = statements.span(!_.toLowerCase.startsWith("select")) + val query = rest.head + val teardown = rest.tail + // The tables are temporary views, which only a temporary view can read. + setup.map(_.replaceFirst("(?i)^create view", "create temp view")).foreach(sql) + try { + // Spark's own answer. Comet on the driver would run the whole query in one JVM, in the + // test's small native memory pool, and Spark's broadcast relations don't fit in the test + // JVM's heap either. + val expected = withSQLConf( + CometConf.COMET_ENABLED.key -> "false", + SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") { + sql(query).collect().toSeq + } + onCluster { + val df = sql(query) + val actual = df.collect().toSeq + val plan = df.queryExecution.executedPlan + assert(collect(plan) { case exec: CometNativeExecutorsExec => exec }.nonEmpty, plan) + if (query.toLowerCase.contains("order by")) { + assert(actual == expected, s"q$q returned rows in another order or other rows") + } else { + checkAnswer(df, expected) + } + } + } finally { + teardown.foreach(sql) + } + } + } + + nativeExecutorsTest("broadcast hash join") { + withTable { path => + val left = spark.read.parquet(path).select("id", "k") + val right = spark.read.parquet(path).where("id < 10").select(col("k").as("k2"), col("v")) + checkOnCluster(left.join(right, left("k") === right("k2")).select("id", "v")) + } + } + + nativeExecutorsTest("scalar subquery") { + withTable { path => + spark.read.parquet(path).createOrReplaceTempView("t") + checkOnCluster(sql("select id, v from t where v > (select avg(v) from t where k = 3)")) + } + } + nativeExecutorsTest("a query the cluster cannot run fails, naming the operator") { withTable { path => - onCluster { - // A small join is broadcast, and the prototype doesn't support broadcast exchanges. - val left = spark.read.parquet(path) - val right = spark.read.parquet(path).where("id < 10").select(col("k").as("k2")) - val error = - intercept[Exception](left.join(right, left("k") === right("k2")).collect()) - val messages = Iterator - .iterate[Throwable](error)(_.getCause) - .takeWhile(_ != null) - .map(e => String.valueOf(e.getMessage)) - .toSeq - assert( - messages.exists(message => - message.contains("Comet Native Executors cannot run this query") && - message.contains("CometBroadcastExchange")), - error) + // With Comet's sort off, the sort stays in Spark. + withSQLConf(CometConf.COMET_EXEC_SORT_ENABLED.key -> "false") { + onCluster { + val error = intercept[Exception](spark.read.parquet(path).orderBy("v").collect()) + val messages = Iterator + .iterate[Throwable](error)(_.getCause) + .takeWhile(_ != null) + .map(e => String.valueOf(e.getMessage)) + .toSeq + assert( + messages.exists(message => + message.contains("Comet Native Executors cannot run this query") && + message.contains("Sort")), + error) + } } } } From fe0c96f0475de137f0146709d7ab7e88afb25e63 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 31 Oct 2026 11:56:14 -0600 Subject: [PATCH 14/36] feat: range-partition sorts on Comet Native Executors by sampled split points A range exchange arrives without bounds, which Spark's driver would find by sampling the exchange's input. The scheduler now does the same while it builds the plan: it runs a job of its own over the stage that feeds the exchange, in which a SampleExec keeps a reservoir sample of the sort keys of each partition, reads the sample back, and cuts it into ranges of about the same number of rows. The exchange becomes a range RepartitionExec with those split points, which Ballista plans as a range shuffle, so the sort that reads it runs in as many tasks. A range exchange whose sort keys aren't all columns, or whose sample yields no split points, is still read as one partition. The workspace now pins Ballista's comet-native branch at the commit that keeps a submitted range repartition as a stage boundary. --- native/native-executors/Cargo.lock | 12 +- native/native-executors/README.md | 17 +- native/native-executors/src/assembly.rs | 143 ++++++- native/native-executors/src/codec.rs | 43 ++ native/native-executors/src/frontend.rs | 143 +++++-- native/native-executors/src/lib.rs | 1 + native/native-executors/src/range.rs | 433 ++++++++++++++++++++ native/native-executors/tests/cluster.rs | 30 ++ native/native-executors/tests/common/mod.rs | 23 +- 9 files changed, 776 insertions(+), 69 deletions(-) create mode 100644 native/native-executors/src/range.rs diff --git a/native/native-executors/Cargo.lock b/native/native-executors/Cargo.lock index 0a52da7f50d..1631f425ed3 100644 --- a/native/native-executors/Cargo.lock +++ b/native/native-executors/Cargo.lock @@ -999,7 +999,7 @@ dependencies = [ [[package]] name = "ballista-core" version = "55.0.0" -source = "git+https://github.com/apache/datafusion-ballista?branch=comet-native#2efa802ffd6f454211d516b660a3f9db648bab65" +source = "git+https://github.com/apache/datafusion-ballista?branch=comet-native#6ef9fe84d3fa41837b6876b09fba7da46942df77" dependencies = [ "arrow-flight", "async-trait", @@ -1028,7 +1028,7 @@ dependencies = [ [[package]] name = "ballista-executor" version = "55.0.0" -source = "git+https://github.com/apache/datafusion-ballista?branch=comet-native#2efa802ffd6f454211d516b660a3f9db648bab65" +source = "git+https://github.com/apache/datafusion-ballista?branch=comet-native#6ef9fe84d3fa41837b6876b09fba7da46942df77" dependencies = [ "arrow", "arrow-flight", @@ -1057,7 +1057,7 @@ dependencies = [ [[package]] name = "ballista-scheduler" version = "55.0.0" -source = "git+https://github.com/apache/datafusion-ballista?branch=comet-native#2efa802ffd6f454211d516b660a3f9db648bab65" +source = "git+https://github.com/apache/datafusion-ballista?branch=comet-native#6ef9fe84d3fa41837b6876b09fba7da46942df77" dependencies = [ "arrow-flight", "async-trait", @@ -5073,8 +5073,8 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ - "heck 0.5.0", - "itertools 0.14.0", + "heck 0.4.1", + "itertools 0.13.0", "log", "multimap", "petgraph", @@ -5095,7 +5095,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.119", diff --git a/native/native-executors/README.md b/native/native-executors/README.md index b332496b031..5af9d2b2d1f 100644 --- a/native/native-executors/README.md +++ b/native/native-executors/README.md @@ -48,15 +48,22 @@ A client runs a query through the scheduler's Arrow Flight service: A stage that several other stages read runs once for each of them, since Ballista's planner doesn't share a stage between consumers yet. +A range exchange needs split points, which Spark's driver would find by sampling the exchange's +input. The scheduler does the same while it builds the plan: it runs a job of its own over the +stage that feeds the exchange, which keeps a reservoir sample of the sort keys of each partition, +and cuts the sorted sample into ranges of about the same number of rows. Like Spark's, the sampling +job runs that stage once more. A range exchange whose sort keys aren't all columns is read as one +partition instead, so its sort runs in a single task. + Each stage runs on the executors as a `CometFragmentExec` (`native/core/src/execution/native_executors`), which plans the stage's block with Comet's planner for each partition. Ballista moves the data between stages: -| Exchange | Ballista plan | -| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| Hash | `RepartitionExec(Hash)`: Ballista's hash shuffle, with DataFusion's hash | -| Single partition, broadcast | `CoalescePartitionsExec` | -| Range | `CoalescePartitionsExec`, so a sort runs in one task ([ballista#2680](https://github.com/apache/datafusion-ballista/issues/2680)) | +| Exchange | Ballista plan | +| --------------------------- | ----------------------------------------------------------------------------------------------------------- | +| Hash | `RepartitionExec(Hash)`: Ballista's hash shuffle, with DataFusion's hash | +| Single partition, broadcast | `CoalescePartitionsExec` | +| Range | `RepartitionExec(Range)`: Ballista's range shuffle, with split points from a sample of the exchange's input | ## Building and running diff --git a/native/native-executors/src/assembly.rs b/native/native-executors/src/assembly.rs index f8ad3e3af0b..ed7f44e5175 100644 --- a/native/native-executors/src/assembly.rs +++ b/native/native-executors/src/assembly.rs @@ -23,19 +23,29 @@ use std::sync::Arc; use comet::execution::native_executors::{ block_schema, partition_descriptor, CometFragmentExec, COMET_VERSION, }; +use datafusion::arrow::array::RecordBatch; +use datafusion::arrow::compute::SortOptions; use datafusion::common::{plan_datafusion_err, plan_err, Result}; use datafusion::physical_expr::expressions::Column; -use datafusion::physical_expr::PhysicalExpr; +use datafusion::physical_expr::{LexOrdering, PhysicalExpr, PhysicalSortExpr, RangePartitioning}; use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; use datafusion::physical_plan::repartition::RepartitionExec; use datafusion::physical_plan::ExecutionPlanProperties; use datafusion::physical_plan::{ExecutionPlan, Partitioning}; use datafusion_comet_proto::spark_expression::expr::ExprStruct; +use datafusion_comet_proto::spark_expression::{Expr, NullOrdering, SortDirection}; use datafusion_comet_proto::spark_native_executors::{Fragment, QueryRequest, Stage, StageInput}; use datafusion_comet_proto::spark_partitioning::partitioning::PartitioningStruct; use datafusion_comet_proto::PROTO_FINGERPRINT; +use log::{info, warn}; use prost::Message; +use crate::range::{sample_size_per_partition, split_points, SampleExec}; + +/// Runs a plan with one output partition to completion and returns its rows: how the scheduler +/// samples the input of a range exchange. +pub type Sampler<'a> = dyn FnMut(Arc) -> Result> + 'a; + /// Checks that `request` comes from a driver running this Comet build. The plan format is not /// yet stable across versions, and an executor would ignore any field it doesn't know. pub fn check_version(request: &QueryRequest) -> Result<()> { @@ -57,10 +67,29 @@ pub fn check_version(request: &QueryRequest) -> Result<()> { /// Each stage's block is planned with Comet's planner to check it before anything runs. That can /// block, so don't call this on an async runtime's thread. A stage that several stages read runs /// once for each of them, since Ballista's planner doesn't share a stage between consumers yet. +/// +/// A range exchange is read as one partition, so the stage that sorts it runs in a single task. +/// [`build_plan_with_sampler`] range-partitions it instead. pub fn build_plan(request: QueryRequest) -> Result> { + build(request, None) +} + +/// Builds the physical plan of `request` like [`build_plan`], but range-partitions each range +/// exchange by split points found by sampling its input with `sampler`, as Spark does. +pub fn build_plan_with_sampler( + request: QueryRequest, + sampler: &mut Sampler, +) -> Result> { + build(request, Some(sampler)) +} + +fn build( + request: QueryRequest, + mut sampler: Option<&mut Sampler>, +) -> Result> { let mut stages: Vec> = Vec::with_capacity(request.stages.len()); for (index, stage) in request.stages.into_iter().enumerate() { - let fragment = build_stage(stage, &stages, &request.conf) + let fragment = build_stage(stage, &stages, &request.conf, sampler.as_deref_mut()) .map_err(|e| e.context(format!("Stage {index} of the query")))?; stages.push(fragment); } @@ -73,6 +102,7 @@ fn build_stage( stage: Stage, producers: &[Arc], conf: &HashMap, + mut sampler: Option<&mut Sampler>, ) -> Result> { let block = stage .plan @@ -87,7 +117,11 @@ fn build_stage( input.producer ) })?; - inputs.push(exchange(input, Arc::clone(producer))?); + inputs.push(exchange( + input, + Arc::clone(producer), + sampler.as_deref_mut(), + )?); } // A stage that reads exchanges has as many partitions as they deliver, which // `CometFragmentExec` checks they agree on. @@ -149,6 +183,7 @@ fn build_stage( fn exchange( input: &StageInput, producer: Arc, + sampler: Option<&mut Sampler>, ) -> Result> { if input.broadcast { return Ok(Arc::new(CoalescePartitionsExec::new(producer))); @@ -163,15 +198,14 @@ fn exchange( let keys = hash .hash_expression .iter() - .map(|expr| match expr.expr_struct.as_ref() { - Some(ExprStruct::Bound(bound)) - if (bound.index as usize) < schema.fields().len() => - { - let index = bound.index as usize; - Ok(Arc::new(Column::new(schema.field(index).name(), index)) - as Arc) - } - _ => plan_err!("Hash partitioning keys must be columns of the stage's input"), + .map(|expr| { + let index = bound_column(expr, schema.fields().len()).ok_or_else(|| { + plan_datafusion_err!( + "Hash partitioning keys must be columns of the stage's input" + ) + })?; + Ok(Arc::new(Column::new(schema.field(index).name(), index)) + as Arc) }) .collect::>>()?; Ok(Arc::new(RepartitionExec::try_new( @@ -179,13 +213,88 @@ fn exchange( Partitioning::Hash(keys, hash.num_partitions as usize), )?)) } - // Ballista plans a range shuffle only for a range partitioning with split points - // (https://github.com/apache/datafusion-ballista/pull/2722), and the driver doesn't - // sample its input for them yet, so the consuming stage reads a range exchange as one - // partition too. That sorts correctly, in a single task. - Some(PartitioningStruct::SinglePartition(_) | PartitioningStruct::RangePartition(_)) => { + Some(PartitioningStruct::RangePartition(range)) => match sampler { + Some(sampler) => range_exchange(range, producer, sampler), + None => Ok(Arc::new(CoalescePartitionsExec::new(producer))), + }, + Some(PartitioningStruct::SinglePartition(_)) => { Ok(Arc::new(CoalescePartitionsExec::new(producer))) } other => plan_err!("Comet Native Executors don't support this exchange yet: {other:?}"), } } + +/// The index of the column `expr` reads, if it reads a column of an input with `num_fields`. +fn bound_column(expr: &Expr, num_fields: usize) -> Option { + match expr.expr_struct.as_ref() { + Some(ExprStruct::Bound(bound)) if (bound.index as usize) < num_fields => { + Some(bound.index as usize) + } + _ => None, + } +} + +/// A range exchange as a range shuffle (`RepartitionExec` with a range partitioning), with +/// split points found by sampling `producer`. It falls back to reading the exchange as one +/// partition, which sorts correctly in a single task, when a sort key is not a column or the +/// sample yields no split points. +fn range_exchange( + range: &datafusion_comet_proto::spark_partitioning::RangePartition, + producer: Arc, + sampler: &mut Sampler, +) -> Result> { + let schema = producer.schema(); + let mut keys = Vec::with_capacity(range.sort_orders.len()); + let mut options = Vec::with_capacity(range.sort_orders.len()); + for order in &range.sort_orders { + let key = match order.expr_struct.as_ref() { + Some(ExprStruct::SortOrder(order)) => order.child.as_deref().and_then(|child| { + bound_column(child, schema.fields().len()).map(|index| (index, order)) + }), + _ => None, + }; + let Some((index, order)) = key else { + warn!("A range exchange sorts by an expression that is not a column, so it runs in one task"); + return Ok(Arc::new(CoalescePartitionsExec::new(producer))); + }; + keys.push(index); + options.push(SortOptions { + descending: order.direction == SortDirection::Descending as i32, + nulls_first: order.null_ordering == NullOrdering::NullsFirst as i32, + }); + } + let partitions = range.num_partitions.max(1) as usize; + if keys.is_empty() || partitions == 1 { + return Ok(Arc::new(CoalescePartitionsExec::new(producer))); + } + + let per_partition = + sample_size_per_partition(partitions, producer.output_partitioning().partition_count()); + let sample = Arc::new(SampleExec::try_new( + Arc::clone(&producer), + keys.clone(), + per_partition, + )?); + let sample_schema = sample.schema(); + let samples = sampler(Arc::new(CoalescePartitionsExec::new(sample)))?; + let points = split_points(&sample_schema, &samples, &options, partitions)?; + if points.is_empty() { + return Ok(Arc::new(CoalescePartitionsExec::new(producer))); + } + info!( + "Range-partitioning an exchange into {} partitions by a sample of {} rows", + points.len() + 1, + samples.iter().map(RecordBatch::num_rows).sum::() + ); + let ordering = LexOrdering::new(keys.iter().zip(&options).map(|(index, options)| { + PhysicalSortExpr::new( + Arc::new(Column::new(schema.field(*index).name(), *index)), + *options, + ) + })) + .ok_or_else(|| plan_datafusion_err!("A range exchange has no sort keys"))?; + Ok(Arc::new(RepartitionExec::try_new( + producer, + Partitioning::Range(RangePartitioning::try_new(ordering, points)?), + )?)) +} diff --git a/native/native-executors/src/codec.rs b/native/native-executors/src/codec.rs index b8b9a7f9320..a0dd469c47d 100644 --- a/native/native-executors/src/codec.rs +++ b/native/native-executors/src/codec.rs @@ -21,6 +21,7 @@ use std::sync::Arc; use ballista_core::serde::BallistaPhysicalExtensionCodec; use comet::execution::native_executors::CometFragmentExec; +use datafusion::common::plan_datafusion_err; use datafusion::common::{internal_datafusion_err, Result}; use datafusion::execution::TaskContext; use datafusion::physical_plan::ExecutionPlan; @@ -28,10 +29,45 @@ use datafusion_comet_proto::spark_native_executors::Fragment; use datafusion_proto::physical_plan::{PhysicalExtensionCodec, PhysicalProtoConverterExtension}; use prost::Message; +use crate::range::SampleExec; + /// Prefix of an encoded [`CometFragmentExec`], which tells it apart from Ballista's own nodes. /// The rest is its `Fragment`; its children travel separately, as the plan's. const FRAGMENT_MAGIC: &[u8] = b"COMET_FRAGMENT\0"; +/// Prefix of an encoded [`SampleExec`]. The rest is its sample size per partition, its number of +/// keys and its keys, each a little-endian `u64`. +const SAMPLE_MAGIC: &[u8] = b"COMET_SAMPLE\0"; + +fn encode_sample(sample: &SampleExec, buf: &mut Vec) { + buf.extend_from_slice(SAMPLE_MAGIC); + buf.extend_from_slice(&(sample.per_partition() as u64).to_le_bytes()); + buf.extend_from_slice(&(sample.keys().len() as u64).to_le_bytes()); + for key in sample.keys() { + buf.extend_from_slice(&(*key as u64).to_le_bytes()); + } +} + +fn decode_sample(buf: &[u8], inputs: &[Arc]) -> Result> { + let (words, rest) = buf.as_chunks::<8>(); + let mut words = words.iter().map(|word| u64::from_le_bytes(*word) as usize); + let bad = || plan_datafusion_err!("Cannot decode a SampleExec"); + let per_partition = words.next().ok_or_else(bad)?; + let num_keys = words.next().ok_or_else(bad)?; + let keys: Vec = words.collect(); + let [input] = inputs else { + return Err(bad()); + }; + if keys.len() != num_keys || !rest.is_empty() { + return Err(bad()); + } + Ok(Arc::new(SampleExec::try_new( + Arc::clone(input), + keys, + per_partition, + )?)) +} + /// Ballista's physical extension codec, extended with Comet's operators. #[derive(Debug, Default)] pub struct CometPhysicalCodec { @@ -46,6 +82,9 @@ impl PhysicalExtensionCodec for CometPhysicalCodec { ctx: &TaskContext, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result> { + if let Some(buf) = buf.strip_prefix(SAMPLE_MAGIC) { + return decode_sample(buf, inputs); + } let Some(buf) = buf.strip_prefix(FRAGMENT_MAGIC) else { return self.ballista.try_decode(buf, inputs, ctx, proto_converter); }; @@ -63,6 +102,10 @@ impl PhysicalExtensionCodec for CometPhysicalCodec { buf: &mut Vec, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result<()> { + if let Some(sample) = node.downcast_ref::() { + encode_sample(sample, buf); + return Ok(()); + } let Some(fragment) = node.downcast_ref::() else { return self.ballista.try_encode(node, buf, proto_converter); }; diff --git a/native/native-executors/src/frontend.rs b/native/native-executors/src/frontend.rs index b6077669819..9380f94cd7f 100644 --- a/native/native-executors/src/frontend.rs +++ b/native/native-executors/src/frontend.rs @@ -26,6 +26,7 @@ use std::sync::Arc; use std::time::{Duration, UNIX_EPOCH}; +use arrow_flight::flight_service_client::FlightServiceClient; use arrow_flight::flight_service_server::FlightService; use arrow_flight::{ Action, ActionType, Criteria, Empty, FlightData, FlightDescriptor, FlightEndpoint, FlightInfo, @@ -35,17 +36,20 @@ use ballista_core::extension::SessionConfigExt; use ballista_core::serde::protobuf::{job_status, PartitionLocation, SuccessfulJob}; use ballista_core::serde::scheduler::{Action as BallistaAction, ShuffleFileKind}; use ballista_scheduler::scheduler_server::SchedulerServer; +use datafusion::arrow::array::RecordBatch; +use datafusion::common::DataFusionError; use datafusion::physical_plan::ExecutionPlan; use datafusion::prelude::SessionConfig; use datafusion_comet_proto::spark_native_executors::QueryRequest; use datafusion_proto::protobuf::{LogicalPlanNode, PhysicalPlanNode}; use futures::stream::BoxStream; +use futures::TryStreamExt; use log::info; use prost::Message; use prost_types::Timestamp; use tonic::{Request, Response, Status, Streaming}; -use crate::assembly::{build_plan, check_version}; +use crate::assembly::{build_plan_with_sampler, check_version}; /// The Ballista session every query runs in. Every query uses Ballista's default settings, and /// executors cache a runtime per session, so sharing one lets queries share those runtimes. @@ -65,41 +69,7 @@ impl CometFlightService { /// Runs `plan` to completion and returns its result: where its output partitions are, in /// partition order, and when it ended. async fn run(&self, plan: Arc) -> Result { - let ctx = self - .scheduler - .state - .session_manager - .create_or_update_session(SESSION_ID, &SessionConfig::new_with_ballista()) - .await - .map_err(|e| Status::internal(format!("Cannot create a session: {e}")))?; - let (sender, mut receiver) = tokio::sync::mpsc::channel(16); - let job_id = self - .scheduler - .submit_physical_plan("comet", ctx, plan, Some(sender)) - .await - .map_err(|e| Status::internal(format!("Cannot submit the query: {e}")))?; - info!("Submitted query as job {job_id}"); - while let Some(status) = receiver.recv().await { - match status.status { - Some(job_status::Status::Successful(mut job)) => { - job.partition_location.sort_by_key(|location| { - let partition = location.partition_id.as_ref().map(|p| p.partition_id); - (partition, location.map_partition_id) - }); - return Ok(job); - } - Some(job_status::Status::Failed(job)) => { - return Err(Status::internal(format!( - "Job {job_id} failed: {}", - job.error - ))) - } - _ => {} - } - } - Err(Status::internal(format!( - "Job {job_id} ended without a final status" - ))) + run(&self.scheduler, plan).await } /// When the executors delete the results of a query that ended at `ended_at`, in milliseconds @@ -122,6 +92,88 @@ impl CometFlightService { } } +/// Runs `plan`, which has one output partition, as a Ballista job on `scheduler`, and reads its +/// rows back from the executor that holds them, as a driver reads a query's result. +async fn collect( + scheduler: &SchedulerServer, + plan: Arc, +) -> Result, Status> { + let job = run(scheduler, plan).await?; + let mut batches = vec![]; + for location in &job.partition_location { + let endpoint = endpoint(location, None)?; + let uri = endpoint + .location + .first() + .map(|location| location.uri.replacen("grpc+tcp://", "http://", 1)) + .ok_or_else(|| Status::internal("A sample partition has no location"))?; + let channel = tonic::transport::Endpoint::from_shared(uri.clone()) + .map_err(|e| Status::internal(format!("Invalid executor location {uri}: {e}")))? + .connect() + .await + .map_err(|e| Status::internal(format!("Cannot connect to {uri}: {e}")))?; + let mut client = arrow_flight::FlightClient::new_from_inner( + FlightServiceClient::new(channel) + .max_decoding_message_size(usize::MAX) + .max_encoding_message_size(usize::MAX), + ); + let ticket = endpoint + .ticket + .ok_or_else(|| Status::internal("A sample partition has no ticket"))?; + let stream = client + .do_get(ticket) + .await + .map_err(|e| Status::internal(format!("Cannot read a sample from {uri}: {e}")))?; + batches.extend( + stream + .try_collect::>() + .await + .map_err(|e| Status::internal(format!("Cannot read a sample from {uri}: {e}")))?, + ); + } + Ok(batches) +} + +/// Runs `plan` as a Ballista job on `scheduler` and waits for it to end. +async fn run( + scheduler: &SchedulerServer, + plan: Arc, +) -> Result { + let ctx = scheduler + .state + .session_manager + .create_or_update_session(SESSION_ID, &SessionConfig::new_with_ballista()) + .await + .map_err(|e| Status::internal(format!("Cannot create a session: {e}")))?; + let (sender, mut receiver) = tokio::sync::mpsc::channel(16); + let job_id = scheduler + .submit_physical_plan("comet", ctx, plan, Some(sender)) + .await + .map_err(|e| Status::internal(format!("Cannot submit the query: {e}")))?; + info!("Submitted query as job {job_id}"); + while let Some(status) = receiver.recv().await { + match status.status { + Some(job_status::Status::Successful(mut job)) => { + job.partition_location.sort_by_key(|location| { + let partition = location.partition_id.as_ref().map(|p| p.partition_id); + (partition, location.map_partition_id) + }); + return Ok(job); + } + Some(job_status::Status::Failed(job)) => { + return Err(Status::internal(format!( + "Job {job_id} failed: {}", + job.error + ))) + } + _ => {} + } + } + Err(Status::internal(format!( + "Job {job_id} ended without a final status" + ))) +} + /// The Flight endpoint for a result partition: the executor's location, a ticket the executor's /// Flight service reads as a Ballista partition fetch, and when the partition is deleted. fn endpoint( @@ -181,10 +233,21 @@ impl FlightService for CometFlightService { let query = QueryRequest::decode(descriptor.cmd.as_ref()) .map_err(|e| Status::invalid_argument(format!("Cannot decode the query: {e}")))?; check_version(&query).map_err(|e| Status::failed_precondition(e.to_string()))?; - let plan = tokio::task::spawn_blocking(move || build_plan(query)) - .await - .map_err(|e| Status::internal(e.to_string()))? - .map_err(|e| Status::invalid_argument(e.to_string()))?; + // A range exchange is sampled by a job of its own while the query is planned, which + // blocks the planning thread on this runtime. + let scheduler = self.scheduler.clone(); + let runtime = tokio::runtime::Handle::current(); + let plan = tokio::task::spawn_blocking(move || { + let mut sampler = |plan| { + runtime + .block_on(collect(&scheduler, plan)) + .map_err(|e| DataFusionError::External(Box::new(e))) + }; + build_plan_with_sampler(query, &mut sampler) + }) + .await + .map_err(|e| Status::internal(e.to_string()))? + .map_err(|e| Status::invalid_argument(e.to_string()))?; let job = self.run(plan).await?; let expires_at = self.results_expire_at(job.ended_at); let mut info = FlightInfo::new(); diff --git a/native/native-executors/src/lib.rs b/native/native-executors/src/lib.rs index 02251c83cb2..ff0543d2b10 100644 --- a/native/native-executors/src/lib.rs +++ b/native/native-executors/src/lib.rs @@ -31,6 +31,7 @@ use std::str::FromStr; pub mod assembly; pub mod codec; pub mod frontend; +pub mod range; /// The value of the environment variable `key`, if it is set, for the binaries' settings. pub fn env_var(key: &str) -> Result, String> diff --git a/native/native-executors/src/range.rs b/native/native-executors/src/range.rs new file mode 100644 index 00000000000..25f993f34e4 --- /dev/null +++ b/native/native-executors/src/range.rs @@ -0,0 +1,433 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Range exchanges: the split points of a range shuffle, found by sampling the exchange's input +//! as Spark's `RangePartitioner` does. +//! +//! The scheduler runs a sampling job over the stage that feeds a range exchange: a +//! [`SampleExec`] over each of its partitions, gathered to one partition. It then sorts the +//! sample by the exchange's sort keys and takes evenly spaced rows as the split points of a +//! [`RangePartitioning`](datafusion::physical_expr::Partitioning::Range), which Ballista plans as a +//! range shuffle. Like Spark's, the sampling job runs the producing stage once more. + +use std::fmt; +use std::sync::Arc; + +use datafusion::arrow::array::RecordBatch; +use datafusion::arrow::compute::{ + concat_batches, interleave_record_batch, lexsort_to_indices, take_record_batch, SortColumn, + SortOptions, +}; +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::arrow::row::{RowConverter, SortField}; +use datafusion::common::tree_node::TreeNodeRecursion; +use datafusion::common::{internal_datafusion_err, Result, ScalarValue, SplitPoint}; +use datafusion::execution::TaskContext; +use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr}; +use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning, + PlanProperties, SendableRecordBatchStream, +}; +use futures::StreamExt; + +/// How many rows to sample from each of `input_partitions` partitions to range-partition their +/// rows into `output_partitions`, as Spark's `RangePartitioner` decides. +pub fn sample_size_per_partition(output_partitions: usize, input_partitions: usize) -> usize { + let sample_size = (20 * output_partitions).min(1_000_000); + (3 * sample_size).div_ceil(input_partitions.max(1)) +} + +/// A uniform sample of up to `per_partition` rows of each partition of its input, keeping only +/// the columns `keys`, by reservoir sampling. +#[derive(Debug)] +pub struct SampleExec { + input: Arc, + keys: Vec, + per_partition: usize, + schema: SchemaRef, + props: Arc, +} + +impl SampleExec { + pub fn try_new( + input: Arc, + keys: Vec, + per_partition: usize, + ) -> Result { + let schema = Arc::new(input.schema().project(&keys)?); + let props = Arc::new(PlanProperties::new( + EquivalenceProperties::new(Arc::clone(&schema)), + Partitioning::UnknownPartitioning(input.output_partitioning().partition_count()), + EmissionType::Final, + Boundedness::Bounded, + )); + Ok(Self { + input, + keys, + per_partition, + schema, + props, + }) + } + + pub fn keys(&self) -> &[usize] { + &self.keys + } + + pub fn per_partition(&self) -> usize { + self.per_partition + } +} + +impl DisplayAs for SampleExec { + fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "SampleExec: keys={:?}, per_partition={}", + self.keys, self.per_partition + ) + } +} + +impl ExecutionPlan for SampleExec { + fn name(&self) -> &str { + "SampleExec" + } + + fn properties(&self) -> &Arc { + &self.props + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.input] + } + + /// The keys are column indexes, so there are no expressions to visit. + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> Result, + ) -> Result { + Ok(TreeNodeRecursion::Continue) + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + let [input] = <[_; 1]>::try_from(children) + .map_err(|_| internal_datafusion_err!("SampleExec takes one input"))?; + Ok(Arc::new(Self::try_new( + input, + self.keys.clone(), + self.per_partition, + )?)) + } + + fn execute( + &self, + partition: usize, + context: Arc, + ) -> Result { + let mut input = self.input.execute(partition, context)?; + let keys = self.keys.clone(); + let capacity = self.per_partition; + let schema = Arc::clone(&self.schema); + let stream = futures::stream::once(async move { + let mut reservoir = RecordBatch::new_empty(Arc::clone(&schema)); + let mut rng = XorShift::new(partition as u64); + // Rows of the partition seen so far. + let mut seen: u64 = 0; + while let Some(batch) = input.next().await { + let batch = batch?.project(&keys)?; + // Where each row of the new reservoir comes from: (0, row) of the reservoir or + // (1, row) of the batch. + let mut indices: Vec<(usize, usize)> = + (0..reservoir.num_rows()).map(|row| (0, row)).collect(); + let mut changed = false; + for row in 0..batch.num_rows() { + if indices.len() < capacity { + indices.push((1, row)); + changed = true; + } else { + let slot = rng.below(seen + 1) as usize; + if slot < capacity { + indices[slot] = (1, row); + changed = true; + } + } + seen += 1; + } + if changed { + reservoir = interleave_record_batch(&[&reservoir, &batch], &indices)?; + } + } + Ok(reservoir) + }); + Ok(Box::pin(RecordBatchStreamAdapter::new( + Arc::clone(&self.schema), + stream, + ))) + } +} + +/// A small deterministic random number generator, so that a query samples the same rows each +/// time it runs. +struct XorShift(u64); + +impl XorShift { + fn new(seed: u64) -> Self { + // Any nonzero state works; mix the seed so that neighbouring partitions differ. + Self(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1) + } + + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + + /// A number in `0..bound`. + fn below(&mut self, bound: u64) -> u64 { + self.next() % bound + } +} + +/// The split points that cut `samples`, rows of the exchange's sort keys, into `partitions` +/// ranges of about the same number of rows, in the order `options` gives each key. Equal +/// candidates collapse into one, so there can be fewer than `partitions - 1`, and none when the +/// sample is empty. +pub fn split_points( + schema: &SchemaRef, + samples: &[RecordBatch], + options: &[SortOptions], + partitions: usize, +) -> Result> { + let batch = concat_batches(schema, samples)?; + let rows = batch.num_rows(); + if rows == 0 || partitions <= 1 { + return Ok(vec![]); + } + let columns: Vec = batch + .columns() + .iter() + .zip(options) + .map(|(values, options)| SortColumn { + values: Arc::clone(values), + options: Some(*options), + }) + .collect(); + let sorted = take_record_batch(&batch, &lexsort_to_indices(&columns, None)?)?; + let converter = RowConverter::new( + schema + .fields() + .iter() + .zip(options) + .map(|(field, options)| { + SortField::new_with_options(field.data_type().clone(), *options) + }) + .collect(), + )?; + let keys = converter.convert_columns(sorted.columns())?; + let mut points = vec![]; + let mut last = None; + for i in 1..partitions { + let index = i * rows / partitions; + let key = keys.row(index); + // Split points must be strictly ordered. + if last.is_some_and(|last| key <= keys.row(last)) { + continue; + } + last = Some(index); + let values = sorted + .columns() + .iter() + .map(|column| ScalarValue::try_from_array(column, index)) + .collect::>>()?; + points.push(SplitPoint::new(values)); + } + Ok(points) +} + +#[cfg(test)] +mod tests { + use super::*; + use datafusion::arrow::array::{Int64Array, StringArray}; + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::datasource::memory::MemorySourceConfig; + use datafusion::physical_plan::collect; + use datafusion::prelude::SessionContext; + + fn longs(values: Vec>) -> (SchemaRef, RecordBatch) { + let schema = Arc::new(Schema::new(vec![Field::new("k", DataType::Int64, true)])); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int64Array::from(values))], + ) + .unwrap(); + (schema, batch) + } + + fn values(points: &[SplitPoint]) -> Vec> { + points.iter().map(|point| point.values().to_vec()).collect() + } + + #[test] + fn split_points_cut_a_sample_into_equal_ranges() { + let (schema, batch) = longs((0..100).rev().map(Some).collect()); + let points = split_points(&schema, &[batch], &[SortOptions::default()], 4).unwrap(); + assert_eq!( + values(&points), + [25, 50, 75].map(|v| vec![ScalarValue::Int64(Some(v))]) + ); + } + + #[test] + fn split_points_follow_the_sort_direction_and_null_order() { + let mut keys: Vec> = (0..6).map(Some).collect(); + keys.extend([None, None]); + let (schema, batch) = longs(keys); + let descending = SortOptions { + descending: true, + nulls_first: true, + }; + let points = split_points(&schema, &[batch], &[descending], 4).unwrap(); + // Sorted: null, null, 5, 4, 3, 2, 1, 0. + assert_eq!( + values(&points), + [Some(5), Some(3), Some(1)].map(|v| vec![ScalarValue::Int64(v)]) + ); + } + + #[test] + fn equal_candidates_collapse_into_one_split_point() { + let (schema, batch) = longs([1, 1, 1, 1, 1, 1, 2, 2].map(Some).to_vec()); + let points = split_points(&schema, &[batch], &[SortOptions::default()], 4).unwrap(); + assert_eq!( + values(&points), + [1, 2].map(|v| vec![ScalarValue::Int64(Some(v))]) + ); + } + + #[test] + fn an_empty_sample_has_no_split_points() { + let (schema, _) = longs(vec![]); + let points = split_points(&schema, &[], &[SortOptions::default()], 4).unwrap(); + assert!(points.is_empty()); + } + + #[test] + fn split_points_order_compound_keys_lexicographically() { + let schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Utf8, false), + Field::new("b", DataType::Int64, false), + ])); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(StringArray::from(vec!["x", "x", "y", "y"])), + Arc::new(Int64Array::from(vec![2, 1, 2, 1])), + ], + ) + .unwrap(); + let options = [ + SortOptions::default(), + SortOptions { + descending: true, + nulls_first: false, + }, + ]; + let points = split_points(&schema, &[batch], &options, 2).unwrap(); + // Sorted: (x, 2), (x, 1), (y, 2), (y, 1). + assert_eq!( + values(&points), + [vec![ + ScalarValue::Utf8(Some("y".to_string())), + ScalarValue::Int64(Some(2)) + ]] + ); + } + + #[tokio::test] + async fn sample_keeps_up_to_its_size_from_each_partition() { + let schema = Arc::new(Schema::new(vec![ + Field::new("k", DataType::Int64, false), + Field::new("v", DataType::Utf8, false), + ])); + let batch = |start: i64| { + RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(Int64Array::from_iter_values(start..start + 50)), + Arc::new(StringArray::from_iter_values( + (start..start + 50).map(|v| v.to_string()), + )), + ], + ) + .unwrap() + }; + // Two partitions: one of 100 rows in two batches, and one of 3 rows. + let partitions = vec![vec![batch(0), batch(50)], vec![batch(1000).slice(0, 3)]]; + let input = + MemorySourceConfig::try_new_exec(&partitions, Arc::clone(&schema), None).unwrap(); + let sample: Arc = + Arc::new(SampleExec::try_new(input, vec![0], 10).unwrap()); + let ctx = SessionContext::new(); + + let mut sizes = vec![]; + for partition in 0..2 { + let batches = datafusion::physical_plan::common::collect( + sample.execute(partition, ctx.task_ctx()).unwrap(), + ) + .await + .unwrap(); + let rows: Vec = batches + .iter() + .flat_map(|batch| { + assert_eq!(batch.num_columns(), 1); + batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .to_vec() + }) + .collect(); + let mut distinct = rows.clone(); + distinct.sort(); + distinct.dedup(); + assert_eq!(distinct.len(), rows.len(), "{rows:?} repeats a row"); + sizes.push(rows.len()); + } + assert_eq!(sizes, [10, 3]); + // The whole plan collects the same rows. + assert_eq!( + collect(sample, ctx.task_ctx()) + .await + .unwrap() + .iter() + .map(RecordBatch::num_rows) + .sum::(), + 13 + ); + } +} diff --git a/native/native-executors/tests/cluster.rs b/native/native-executors/tests/cluster.rs index dee8dcaa7b5..aa7da7f2609 100644 --- a/native/native-executors/tests/cluster.rs +++ b/native/native-executors/tests/cluster.rs @@ -207,6 +207,36 @@ async fn runs_a_query_across_a_range_exchange() { ); } +/// The scheduler samples the producer for split points, so each partition holds one range of +/// keys, in the exchange's order. +#[tokio::test(flavor = "multi_thread")] +async fn range_partitions_a_range_exchange_by_sampled_split_points() { + let cluster = Cluster::start(); + let request = query(vec![ + range_stage(1000, 4), + reading_stage(0, range_on_first_column_descending(4), 4), + ]); + let partitions = cluster.run(&request).await; + assert_eq!(partitions.len(), 4); + let ranges: Vec> = partitions + .iter() + .map(|partition| sorted_values::(partition)) + .collect(); + for (index, range) in ranges.iter().enumerate() { + assert!(!range.is_empty(), "partition {index} is empty"); + } + for pair in ranges.windows(2) { + assert!( + pair[0].first() > pair[1].last(), + "a partition holds keys below the next one's" + ); + } + assert_eq!( + sorted_values::(&partitions.concat()), + (0..1000).collect::>() + ); +} + #[tokio::test(flavor = "multi_thread")] async fn refuses_a_query_from_another_build() { let cluster = Cluster::start(); diff --git a/native/native-executors/tests/common/mod.rs b/native/native-executors/tests/common/mod.rs index 5dc6b530cf7..a5fbcb68b12 100644 --- a/native/native-executors/tests/common/mod.rs +++ b/native/native-executors/tests/common/mod.rs @@ -25,7 +25,9 @@ use comet::execution::native_executors::COMET_VERSION; use datafusion::arrow::array::{AsArray, RecordBatch}; use datafusion::arrow::datatypes::ArrowPrimitiveType; use datafusion_comet_proto::spark_expression::data_type::DataTypeId; -use datafusion_comet_proto::spark_expression::{self, expr::ExprStruct, BoundReference, Expr}; +use datafusion_comet_proto::spark_expression::{ + self, expr::ExprStruct, BoundReference, Expr, NullOrdering, SortDirection, SortOrder, +}; use datafusion_comet_proto::spark_native_executors::{QueryRequest, Stage, StageInput}; use datafusion_comet_proto::spark_operator::{ operator::OpStruct, Operator, RangeScan, Scan, SparkStructField, @@ -150,6 +152,25 @@ pub fn range_partitioning(num_partitions: i32) -> Partitioning { } } +/// A range exchange as a driver sends it, sorting by the first column, descending. +pub fn range_on_first_column_descending(num_partitions: i32) -> Partitioning { + let order = Expr { + expr_struct: Some(ExprStruct::SortOrder(Box::new(SortOrder { + child: Some(Box::new(bound(0, long_type()))), + direction: SortDirection::Descending as i32, + null_ordering: NullOrdering::NullsLast as i32, + }))), + ..Default::default() + }; + Partitioning { + partitioning_struct: Some(PartitioningStruct::RangePartition(RangePartition { + sort_orders: vec![order], + num_partitions, + ..Default::default() + })), + } +} + /// The values of the batches' single column, of type `T`, sorted. pub fn sorted_values(batches: &[RecordBatch]) -> Vec where From b4a83e5e29b10b626b926f0a8f7dc75a81a12f44 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 31 Oct 2026 12:02:55 -0600 Subject: [PATCH 15/36] fix: keep float sort keys off range shuffles and report sampling failures as such A range shuffle compares raw values, while Comet's sort normalizes float keys first, so a NaN with its sign bit set would land in the lowest range but sort last. A range exchange now range-partitions only by keys whose types Arrow orders as Spark does, and reads any other as one partition. A failed sampling job now reaches the driver with its own status rather than as an invalid query, the sampler no longer wraps its plan in a coalesce the adaptive planner removes, and a subquery's result, already in Catalyst's internal form, becomes a Literal directly. --- native/native-executors/src/assembly.rs | 39 +++++++++-- native/native-executors/src/frontend.rs | 17 ++++- native/native-executors/tests/assembly.rs | 65 ++++++++++++++++++- native/native-executors/tests/common/mod.rs | 4 ++ .../sql/comet/CometNativeExecutorsExec.scala | 3 +- 5 files changed, 118 insertions(+), 10 deletions(-) diff --git a/native/native-executors/src/assembly.rs b/native/native-executors/src/assembly.rs index ed7f44e5175..dc026135afb 100644 --- a/native/native-executors/src/assembly.rs +++ b/native/native-executors/src/assembly.rs @@ -25,6 +25,7 @@ use comet::execution::native_executors::{ }; use datafusion::arrow::array::RecordBatch; use datafusion::arrow::compute::SortOptions; +use datafusion::arrow::datatypes::DataType; use datafusion::common::{plan_datafusion_err, plan_err, Result}; use datafusion::physical_expr::expressions::Column; use datafusion::physical_expr::{LexOrdering, PhysicalExpr, PhysicalSortExpr, RangePartitioning}; @@ -42,7 +43,7 @@ use prost::Message; use crate::range::{sample_size_per_partition, split_points, SampleExec}; -/// Runs a plan with one output partition to completion and returns its rows: how the scheduler +/// Runs a plan to completion and returns the rows of all its partitions: how the scheduler /// samples the input of a range exchange. pub type Sampler<'a> = dyn FnMut(Arc) -> Result> + 'a; @@ -234,10 +235,35 @@ fn bound_column(expr: &Expr, num_fields: usize) -> Option { } } +/// Whether Arrow orders values of `data_type` as Spark does, so that a range shuffle, which +/// compares raw values, puts rows in the ranges that Comet's sort orders them by. Comet's sort +/// normalizes floats first (a NaN of either sign sorts last), so a float key, or a nested one +/// that may hold floats, can't be range-partitioned this way. +fn orders_like_spark(data_type: &DataType) -> bool { + data_type.is_integer() + || matches!( + data_type, + DataType::Boolean + | DataType::Decimal32(_, _) + | DataType::Decimal64(_, _) + | DataType::Decimal128(_, _) + | DataType::Decimal256(_, _) + | DataType::Date32 + | DataType::Date64 + | DataType::Timestamp(_, _) + | DataType::Utf8 + | DataType::LargeUtf8 + | DataType::Utf8View + | DataType::Binary + | DataType::LargeBinary + | DataType::BinaryView + ) +} + /// A range exchange as a range shuffle (`RepartitionExec` with a range partitioning), with /// split points found by sampling `producer`. It falls back to reading the exchange as one -/// partition, which sorts correctly in a single task, when a sort key is not a column or the -/// sample yields no split points. +/// partition, which sorts correctly in a single task, when a sort key is not a column, or of a +/// type a range shuffle doesn't order as Spark does, or the sample yields no split points. fn range_exchange( range: &datafusion_comet_proto::spark_partitioning::RangePartition, producer: Arc, @@ -257,6 +283,11 @@ fn range_exchange( warn!("A range exchange sorts by an expression that is not a column, so it runs in one task"); return Ok(Arc::new(CoalescePartitionsExec::new(producer))); }; + let data_type = schema.field(index).data_type(); + if !orders_like_spark(data_type) { + warn!("A range exchange sorts by a {data_type} column, so it runs in one task"); + return Ok(Arc::new(CoalescePartitionsExec::new(producer))); + } keys.push(index); options.push(SortOptions { descending: order.direction == SortDirection::Descending as i32, @@ -276,7 +307,7 @@ fn range_exchange( per_partition, )?); let sample_schema = sample.schema(); - let samples = sampler(Arc::new(CoalescePartitionsExec::new(sample)))?; + let samples = sampler(sample)?; let points = split_points(&sample_schema, &samples, &options, partitions)?; if points.is_empty() { return Ok(Arc::new(CoalescePartitionsExec::new(producer))); diff --git a/native/native-executors/src/frontend.rs b/native/native-executors/src/frontend.rs index 9380f94cd7f..14822ed3278 100644 --- a/native/native-executors/src/frontend.rs +++ b/native/native-executors/src/frontend.rs @@ -92,8 +92,8 @@ impl CometFlightService { } } -/// Runs `plan`, which has one output partition, as a Ballista job on `scheduler`, and reads its -/// rows back from the executor that holds them, as a driver reads a query's result. +/// Runs `plan` as a Ballista job on `scheduler` and reads the rows of each of its partitions back +/// from the executor that holds it, as a driver reads a query's result. async fn collect( scheduler: &SchedulerServer, plan: Arc, @@ -209,6 +209,17 @@ fn endpoint( Ok(endpoint) } +/// The status for a query the scheduler couldn't plan: a sampling job's own status when that job +/// failed, and otherwise an invalid query. +fn planning_status(error: DataFusionError) -> Status { + if let DataFusionError::External(cause) = error.find_root() { + if let Some(status) = cause.downcast_ref::() { + return status.clone(); + } + } + Status::invalid_argument(error.to_string()) +} + fn unimplemented(method: &str) -> Result { Err(Status::unimplemented(format!( "A Comet scheduler doesn't implement {method}" @@ -247,7 +258,7 @@ impl FlightService for CometFlightService { }) .await .map_err(|e| Status::internal(e.to_string()))? - .map_err(|e| Status::invalid_argument(e.to_string()))?; + .map_err(planning_status)?; let job = self.run(plan).await?; let expires_at = self.results_expire_at(job.ended_at); let mut info = FlightInfo::new(); diff --git a/native/native-executors/tests/assembly.rs b/native/native-executors/tests/assembly.rs index 63fcd103b9c..66c20ef784a 100644 --- a/native/native-executors/tests/assembly.rs +++ b/native/native-executors/tests/assembly.rs @@ -20,10 +20,12 @@ mod common; -use comet_native_executors::assembly::build_plan; +use comet_native_executors::assembly::{build_plan, build_plan_with_sampler}; use common::*; use datafusion::arrow::datatypes::{DataType, Int32Type, Int64Type}; -use datafusion::physical_plan::{collect, ExecutionPlan, ExecutionPlanProperties}; +use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; +use datafusion::physical_plan::repartition::RepartitionExec; +use datafusion::physical_plan::{collect, ExecutionPlan, ExecutionPlanProperties, Partitioning}; use datafusion::prelude::SessionContext; use datafusion_comet_proto::spark_expression::{ expr::ExprStruct, literal::Value, Expr, IfExpr, Literal, Subquery, @@ -87,6 +89,65 @@ async fn range_exchange_is_read_as_one_partition() { assert_eq!(run(request).await, (0..100).collect::>()); } +/// Assembles `request` with a sampler that runs each sampling plan in this process, and returns +/// the plan and how many sampling plans ran. +async fn assemble_sampling(request: QueryRequest) -> (Arc, usize) { + let runtime = tokio::runtime::Handle::current(); + tokio::task::spawn_blocking(move || { + let mut samples = 0; + let mut sampler = |plan: Arc| { + samples += 1; + runtime.block_on(collect(plan, SessionContext::new().task_ctx())) + }; + let plan = build_plan_with_sampler(request, &mut sampler).unwrap(); + (plan, samples) + }) + .await + .unwrap() +} + +/// The exchange a reading stage reads: its second child, after the partition descriptor. +fn exchange_of(plan: &Arc) -> Arc { + Arc::clone(plan.children()[1]) +} + +#[tokio::test(flavor = "multi_thread")] +async fn range_exchange_is_range_partitioned_by_a_sample() { + let request = query(vec![ + range_stage(1000, 4), + reading_stage(0, range_on_first_column_descending(4), 4), + ]); + let (plan, samples) = assemble_sampling(request).await; + assert_eq!(samples, 1); + assert_eq!(plan.output_partitioning().partition_count(), 4); + let exchange = exchange_of(&plan); + let repartition = exchange + .downcast_ref::() + .expect("a range exchange becomes a RepartitionExec"); + let Partitioning::Range(range) = repartition.partitioning() else { + panic!( + "expected a range partitioning, got {}", + repartition.partitioning() + ); + }; + assert_eq!(range.split_points().len(), 3); + assert!(range.ordering()[0].options.descending); +} + +/// A range shuffle compares raw values, which order floats differently from Comet's sort. +#[tokio::test(flavor = "multi_thread")] +async fn range_exchange_on_a_float_key_is_read_as_one_partition() { + let mut producer = range_stage(1000, 4); + producer.output_schema = vec![field(double_type())]; + let request = query(vec![ + producer, + reading_stage(0, range_on_first_column_descending(4), 4), + ]); + let (plan, samples) = assemble_sampling(request).await; + assert_eq!(samples, 0); + assert!(exchange_of(&plan).is::()); +} + #[tokio::test(flavor = "multi_thread")] async fn partition_plans_must_match_the_partitions() { let mut stage = range_stage(100, 4); diff --git a/native/native-executors/tests/common/mod.rs b/native/native-executors/tests/common/mod.rs index a5fbcb68b12..79fb3ec9e47 100644 --- a/native/native-executors/tests/common/mod.rs +++ b/native/native-executors/tests/common/mod.rs @@ -62,6 +62,10 @@ pub fn int_type() -> spark_expression::DataType { data_type(DataTypeId::Int32) } +pub fn double_type() -> spark_expression::DataType { + data_type(DataTypeId::Double) +} + pub fn bool_type() -> spark_expression::DataType { data_type(DataTypeId::Bool) } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala index e3e602302f9..3c2489c52e4 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala @@ -259,7 +259,8 @@ object CometNativeExecutors { return query } val literals: Map[Long, ExprOuterClass.Expr] = subqueries.map { sub => - val literal = Literal.create(sub.eval(null), sub.dataType) + // The result is already in Catalyst's internal form. + val literal = Literal(sub.eval(null), sub.dataType) val expr = QueryPlanSerde .exprToProto(literal, Seq.empty) .getOrElse(throw new CometRuntimeException( From d3c63b2fff4ec33f7ce8063e2b59b45467b5a9bd Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 31 Oct 2026 12:21:33 -0600 Subject: [PATCH 16/36] refactor: simplify the Comet Native Executors driver and range sampling The scheduler has one build_plan, which takes the sampler, and a range exchange that can't be range-partitioned falls back through one helper. It checks that a stage's exchanges agree on their partition count before the plan reaches Ballista. It reads a sampling job's partitions with Ballista's client, a few at a time, and a SampleExec travels as a Sample message. split_points sorts the sample by the row format alone, and the reservoir allocates only when a batch changes it. The driver stamps a query with its version by appending the fields rather than decoding and encoding it, rewrites a stage's partition plans only when its plan holds a subquery, and walks protobuf messages with one helper. Comments link the follow-on issues under #7288. --- .../src/execution/native_executors/client.rs | 18 +- native/native-executors/src/assembly.rs | 174 ++++++------- native/native-executors/src/codec.rs | 55 ++-- native/native-executors/src/frontend.rs | 111 ++++---- native/native-executors/src/range.rs | 63 +++-- native/native-executors/tests/assembly.rs | 56 ++-- native/native-executors/tests/cluster.rs | 4 +- native/native-executors/tests/common/mod.rs | 18 +- native/proto/src/proto/native_executors.proto | 7 + .../sql/comet/CometNativeExecutorsExec.scala | 241 +++++++++--------- .../exec/CometNativeExecutorsSuite.scala | 48 ++-- 11 files changed, 404 insertions(+), 391 deletions(-) diff --git a/native/core/src/execution/native_executors/client.rs b/native/core/src/execution/native_executors/client.rs index d268316070e..c78d1ca09bd 100644 --- a/native/core/src/execution/native_executors/client.rs +++ b/native/core/src/execution/native_executors/client.rs @@ -64,13 +64,19 @@ async fn connect(location: &str) -> CometResult { /// Submits `query`, a serialized `QueryRequest`, to the scheduler at `scheduler_url` and waits /// for it to finish. Returns where its result partitions are. -async fn submit(scheduler_url: &str, query: &[u8]) -> CometResult { - let mut request = QueryRequest::decode(query).map_err(|e| client_error("bad query", e))?; - request.comet_version = COMET_VERSION.to_string(); - request.proto_fingerprint = PROTO_FINGERPRINT.to_string(); +async fn submit(scheduler_url: &str, mut query: Vec) -> CometResult { + // Stamp the query with this build's version and plan format. A field that appears again + // replaces the first, so appending them spares decoding and encoding the whole query. + QueryRequest { + comet_version: COMET_VERSION.to_string(), + proto_fingerprint: PROTO_FINGERPRINT.to_string(), + ..Default::default() + } + .encode(&mut query) + .map_err(|e| client_error("bad query", e))?; let mut client = connect(scheduler_url).await?; let info = client - .get_flight_info(FlightDescriptor::new_cmd(request.encode_to_vec())) + .get_flight_info(FlightDescriptor::new_cmd(query)) .await .map_err(|e| client_error("the query failed", e))?; let endpoints = info @@ -106,7 +112,7 @@ pub extern "system" fn Java_org_apache_comet_NativeExecutorsClient_submit( .try_to_string(env) .map_err(|e| client_error("bad scheduler URL", e))?; let query = env.convert_byte_array(query)?; - let endpoints = get_runtime().block_on(submit(&scheduler_url, &query))?; + let endpoints = get_runtime().block_on(submit(&scheduler_url, query))?; Ok(env .byte_array_from_slice(&endpoints.encode_to_vec())? .into_raw()) diff --git a/native/native-executors/src/assembly.rs b/native/native-executors/src/assembly.rs index dc026135afb..541a3fac3a9 100644 --- a/native/native-executors/src/assembly.rs +++ b/native/native-executors/src/assembly.rs @@ -25,7 +25,7 @@ use comet::execution::native_executors::{ }; use datafusion::arrow::array::RecordBatch; use datafusion::arrow::compute::SortOptions; -use datafusion::arrow::datatypes::DataType; +use datafusion::arrow::datatypes::{DataType, SchemaRef}; use datafusion::common::{plan_datafusion_err, plan_err, Result}; use datafusion::physical_expr::expressions::Column; use datafusion::physical_expr::{LexOrdering, PhysicalExpr, PhysicalSortExpr, RangePartitioning}; @@ -63,34 +63,17 @@ pub fn check_version(request: &QueryRequest) -> Result<()> { } /// Builds the physical plan of `request`: a [`CometFragmentExec`] per stage, with a node that -/// Ballista's planner turns into a stage boundary on each edge. The root is the last stage. +/// Ballista's planner turns into a stage boundary on each edge. The root is the last stage. Each +/// range exchange is range-partitioned by split points found by sampling its input with +/// `sampler`, as Spark does. /// /// Each stage's block is planned with Comet's planner to check it before anything runs. That can /// block, so don't call this on an async runtime's thread. A stage that several stages read runs /// once for each of them, since Ballista's planner doesn't share a stage between consumers yet. -/// -/// A range exchange is read as one partition, so the stage that sorts it runs in a single task. -/// [`build_plan_with_sampler`] range-partitions it instead. -pub fn build_plan(request: QueryRequest) -> Result> { - build(request, None) -} - -/// Builds the physical plan of `request` like [`build_plan`], but range-partitions each range -/// exchange by split points found by sampling its input with `sampler`, as Spark does. -pub fn build_plan_with_sampler( - request: QueryRequest, - sampler: &mut Sampler, -) -> Result> { - build(request, Some(sampler)) -} - -fn build( - request: QueryRequest, - mut sampler: Option<&mut Sampler>, -) -> Result> { +pub fn build_plan(request: QueryRequest, sampler: &mut Sampler) -> Result> { let mut stages: Vec> = Vec::with_capacity(request.stages.len()); for (index, stage) in request.stages.into_iter().enumerate() { - let fragment = build_stage(stage, &stages, &request.conf, sampler.as_deref_mut()) + let fragment = build_stage(stage, &stages, &request.conf, sampler) .map_err(|e| e.context(format!("Stage {index} of the query")))?; stages.push(fragment); } @@ -103,13 +86,12 @@ fn build_stage( stage: Stage, producers: &[Arc], conf: &HashMap, - mut sampler: Option<&mut Sampler>, + sampler: &mut Sampler, ) -> Result> { - let block = stage + let plan = stage .plan .as_ref() - .ok_or_else(|| plan_datafusion_err!("The stage has no plan"))? - .encode_to_vec(); + .ok_or_else(|| plan_datafusion_err!("The stage has no plan"))?; let mut inputs = Vec::with_capacity(stage.inputs.len()); for input in &stage.inputs { let producer = producers.get(input.producer as usize).ok_or_else(|| { @@ -118,21 +100,21 @@ fn build_stage( input.producer ) })?; - inputs.push(exchange( - input, - Arc::clone(producer), - sampler.as_deref_mut(), - )?); + inputs.push(exchange(input, Arc::clone(producer), sampler)?); } - // A stage that reads exchanges has as many partitions as they deliver, which - // `CometFragmentExec` checks they agree on. - let num_partitions = stage + // A stage that reads exchanges has as many partitions as they deliver, and they must agree. + let mut partitioned = stage .inputs .iter() .zip(&inputs) - .find(|(input, _)| !input.broadcast) - .map(|(_, exchange)| exchange.output_partitioning().partition_count()) - .unwrap_or(stage.num_partitions as usize); + .filter(|(input, _)| !input.broadcast) + .map(|(_, exchange)| exchange.output_partitioning().partition_count()); + let num_partitions = partitioned.next().unwrap_or(stage.num_partitions as usize); + if let Some(other) = partitioned.find(|count| *count != num_partitions) { + return plan_err!( + "The stage reads exchanges of {num_partitions} and {other} partitions, which must agree" + ); + } if num_partitions == 0 { return plan_err!("The stage has no partitions"); } @@ -142,8 +124,14 @@ fn build_stage( stage.partition_plans.len() ); } - // A stage that reads files carries its scans' schemas and options only in each partition's - // copy of the block, so check one of those. Checking reads none of its files. + // A stage that reads files runs each partition's own copy of the block, so the block itself + // is only needed for a stage that doesn't. Checking one of those copies reads none of its + // files. + let block = if stage.partition_plans.is_empty() { + plan.encode_to_vec() + } else { + vec![] + }; let planned = block_schema(stage.partition_plans.first().unwrap_or(&block), conf)?; if planned.fields().len() != stage.output_schema.len() { return plan_err!( @@ -159,12 +147,7 @@ fn build_stage( )?]; children.extend(inputs); let fragment = Fragment { - // A stage that reads files runs each partition's own copy of the block instead. - block: if stage.partition_plans.is_empty() { - block - } else { - vec![] - }, + block, broadcast: stage.inputs.iter().map(|input| input.broadcast).collect(), output_schema: stage.output_schema, conf: conf.clone(), @@ -180,14 +163,15 @@ fn build_stage( /// /// A hash exchange partitions with DataFusion's hash function, not Spark's Murmur3. That is /// consistent within a query, which is all joins, aggregates and windows need, but a plan that -/// relies on Spark's own placement of rows, such as a join with a bucketed table, would be wrong. +/// relies on Spark's own placement of rows, such as a join with a bucketed table, would be wrong +/// (https://github.com/apache/datafusion-comet/issues/7368). fn exchange( input: &StageInput, producer: Arc, - sampler: Option<&mut Sampler>, + sampler: &mut Sampler, ) -> Result> { if input.broadcast { - return Ok(Arc::new(CoalescePartitionsExec::new(producer))); + return Ok(one_partition(producer)); } let partitioning = input .partitioning @@ -205,8 +189,7 @@ fn exchange( "Hash partitioning keys must be columns of the stage's input" ) })?; - Ok(Arc::new(Column::new(schema.field(index).name(), index)) - as Arc) + Ok(column(&schema, index)) }) .collect::>>()?; Ok(Arc::new(RepartitionExec::try_new( @@ -214,17 +197,27 @@ fn exchange( Partitioning::Hash(keys, hash.num_partitions as usize), )?)) } - Some(PartitioningStruct::RangePartition(range)) => match sampler { - Some(sampler) => range_exchange(range, producer, sampler), - None => Ok(Arc::new(CoalescePartitionsExec::new(producer))), - }, - Some(PartitioningStruct::SinglePartition(_)) => { - Ok(Arc::new(CoalescePartitionsExec::new(producer))) + Some(PartitioningStruct::RangePartition(range)) => { + Ok(match range_exchange(range, &producer, sampler)? { + Some(partitioning) => Arc::new(RepartitionExec::try_new(producer, partitioning)?), + // Read as one partition, the stage sorts correctly in a single task. + None => one_partition(producer), + }) } + Some(PartitioningStruct::SinglePartition(_)) => Ok(one_partition(producer)), other => plan_err!("Comet Native Executors don't support this exchange yet: {other:?}"), } } +/// `producer`'s output gathered into one partition. +fn one_partition(producer: Arc) -> Arc { + Arc::new(CoalescePartitionsExec::new(producer)) +} + +fn column(schema: &SchemaRef, index: usize) -> Arc { + Arc::new(Column::new(schema.field(index).name(), index)) +} + /// The index of the column `expr` reads, if it reads a column of an input with `num_fields`. fn bound_column(expr: &Expr, num_fields: usize) -> Option { match expr.expr_struct.as_ref() { @@ -238,7 +231,8 @@ fn bound_column(expr: &Expr, num_fields: usize) -> Option { /// Whether Arrow orders values of `data_type` as Spark does, so that a range shuffle, which /// compares raw values, puts rows in the ranges that Comet's sort orders them by. Comet's sort /// normalizes floats first (a NaN of either sign sorts last), so a float key, or a nested one -/// that may hold floats, can't be range-partitioned this way. +/// that may hold floats, can't be range-partitioned this way +/// (https://github.com/apache/datafusion-comet/issues/7366). fn orders_like_spark(data_type: &DataType) -> bool { data_type.is_integer() || matches!( @@ -260,18 +254,25 @@ fn orders_like_spark(data_type: &DataType) -> bool { ) } -/// A range exchange as a range shuffle (`RepartitionExec` with a range partitioning), with -/// split points found by sampling `producer`. It falls back to reading the exchange as one -/// partition, which sorts correctly in a single task, when a sort key is not a column, or of a -/// type a range shuffle doesn't order as Spark does, or the sample yields no split points. +/// The range partitioning of a range exchange over `producer`, with split points found by +/// sampling it. `None` when the exchange should be read as one partition instead: when a sort +/// key is not a column, or of a type a range shuffle doesn't order as Spark does, or the sample +/// yields no split points. +/// +/// The sampling job runs the stages below the exchange once more +/// (https://github.com/apache/datafusion-comet/issues/7367). fn range_exchange( range: &datafusion_comet_proto::spark_partitioning::RangePartition, - producer: Arc, + producer: &Arc, sampler: &mut Sampler, -) -> Result> { +) -> Result> { + let partitions = range.num_partitions.max(1) as usize; + if partitions == 1 { + return Ok(None); + } let schema = producer.schema(); let mut keys = Vec::with_capacity(range.sort_orders.len()); - let mut options = Vec::with_capacity(range.sort_orders.len()); + let mut sort_exprs = Vec::with_capacity(range.sort_orders.len()); for order in &range.sort_orders { let key = match order.expr_struct.as_ref() { Some(ExprStruct::SortOrder(order)) => order.child.as_deref().and_then(|child| { @@ -281,51 +282,46 @@ fn range_exchange( }; let Some((index, order)) = key else { warn!("A range exchange sorts by an expression that is not a column, so it runs in one task"); - return Ok(Arc::new(CoalescePartitionsExec::new(producer))); + return Ok(None); }; let data_type = schema.field(index).data_type(); if !orders_like_spark(data_type) { warn!("A range exchange sorts by a {data_type} column, so it runs in one task"); - return Ok(Arc::new(CoalescePartitionsExec::new(producer))); + return Ok(None); } keys.push(index); - options.push(SortOptions { - descending: order.direction == SortDirection::Descending as i32, - nulls_first: order.null_ordering == NullOrdering::NullsFirst as i32, - }); - } - let partitions = range.num_partitions.max(1) as usize; - if keys.is_empty() || partitions == 1 { - return Ok(Arc::new(CoalescePartitionsExec::new(producer))); + sort_exprs.push(PhysicalSortExpr::new( + column(&schema, index), + SortOptions { + descending: order.direction == SortDirection::Descending as i32, + nulls_first: order.null_ordering == NullOrdering::NullsFirst as i32, + }, + )); } + let options: Vec = sort_exprs.iter().map(|expr| expr.options).collect(); + let Some(ordering) = LexOrdering::new(sort_exprs) else { + return Ok(None); + }; let per_partition = sample_size_per_partition(partitions, producer.output_partitioning().partition_count()); let sample = Arc::new(SampleExec::try_new( - Arc::clone(&producer), - keys.clone(), + Arc::clone(producer), + keys, per_partition, )?); let sample_schema = sample.schema(); let samples = sampler(sample)?; let points = split_points(&sample_schema, &samples, &options, partitions)?; if points.is_empty() { - return Ok(Arc::new(CoalescePartitionsExec::new(producer))); + return Ok(None); } info!( "Range-partitioning an exchange into {} partitions by a sample of {} rows", points.len() + 1, samples.iter().map(RecordBatch::num_rows).sum::() ); - let ordering = LexOrdering::new(keys.iter().zip(&options).map(|(index, options)| { - PhysicalSortExpr::new( - Arc::new(Column::new(schema.field(*index).name(), *index)), - *options, - ) - })) - .ok_or_else(|| plan_datafusion_err!("A range exchange has no sort keys"))?; - Ok(Arc::new(RepartitionExec::try_new( - producer, - Partitioning::Range(RangePartitioning::try_new(ordering, points)?), - )?)) + Ok(Some(Partitioning::Range(RangePartitioning::try_new( + ordering, points, + )?))) } diff --git a/native/native-executors/src/codec.rs b/native/native-executors/src/codec.rs index a0dd469c47d..f0de2b712f6 100644 --- a/native/native-executors/src/codec.rs +++ b/native/native-executors/src/codec.rs @@ -21,11 +21,10 @@ use std::sync::Arc; use ballista_core::serde::BallistaPhysicalExtensionCodec; use comet::execution::native_executors::CometFragmentExec; -use datafusion::common::plan_datafusion_err; use datafusion::common::{internal_datafusion_err, Result}; use datafusion::execution::TaskContext; use datafusion::physical_plan::ExecutionPlan; -use datafusion_comet_proto::spark_native_executors::Fragment; +use datafusion_comet_proto::spark_native_executors::{Fragment, Sample}; use datafusion_proto::physical_plan::{PhysicalExtensionCodec, PhysicalProtoConverterExtension}; use prost::Message; @@ -35,39 +34,9 @@ use crate::range::SampleExec; /// The rest is its `Fragment`; its children travel separately, as the plan's. const FRAGMENT_MAGIC: &[u8] = b"COMET_FRAGMENT\0"; -/// Prefix of an encoded [`SampleExec`]. The rest is its sample size per partition, its number of -/// keys and its keys, each a little-endian `u64`. +/// Prefix of an encoded [`SampleExec`]. The rest is its `Sample`; its input travels separately. const SAMPLE_MAGIC: &[u8] = b"COMET_SAMPLE\0"; -fn encode_sample(sample: &SampleExec, buf: &mut Vec) { - buf.extend_from_slice(SAMPLE_MAGIC); - buf.extend_from_slice(&(sample.per_partition() as u64).to_le_bytes()); - buf.extend_from_slice(&(sample.keys().len() as u64).to_le_bytes()); - for key in sample.keys() { - buf.extend_from_slice(&(*key as u64).to_le_bytes()); - } -} - -fn decode_sample(buf: &[u8], inputs: &[Arc]) -> Result> { - let (words, rest) = buf.as_chunks::<8>(); - let mut words = words.iter().map(|word| u64::from_le_bytes(*word) as usize); - let bad = || plan_datafusion_err!("Cannot decode a SampleExec"); - let per_partition = words.next().ok_or_else(bad)?; - let num_keys = words.next().ok_or_else(bad)?; - let keys: Vec = words.collect(); - let [input] = inputs else { - return Err(bad()); - }; - if keys.len() != num_keys || !rest.is_empty() { - return Err(bad()); - } - Ok(Arc::new(SampleExec::try_new( - Arc::clone(input), - keys, - per_partition, - )?)) -} - /// Ballista's physical extension codec, extended with Comet's operators. #[derive(Debug, Default)] pub struct CometPhysicalCodec { @@ -83,7 +52,16 @@ impl PhysicalExtensionCodec for CometPhysicalCodec { proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result> { if let Some(buf) = buf.strip_prefix(SAMPLE_MAGIC) { - return decode_sample(buf, inputs); + let sample = Sample::decode(buf) + .map_err(|e| internal_datafusion_err!("Cannot decode a SampleExec: {e}"))?; + let [input] = inputs else { + return Err(internal_datafusion_err!("A SampleExec takes one input")); + }; + return Ok(Arc::new(SampleExec::try_new( + Arc::clone(input), + sample.keys.iter().map(|key| *key as usize).collect(), + sample.per_partition as usize, + )?)); } let Some(buf) = buf.strip_prefix(FRAGMENT_MAGIC) else { return self.ballista.try_decode(buf, inputs, ctx, proto_converter); @@ -103,8 +81,13 @@ impl PhysicalExtensionCodec for CometPhysicalCodec { proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result<()> { if let Some(sample) = node.downcast_ref::() { - encode_sample(sample, buf); - return Ok(()); + buf.extend_from_slice(SAMPLE_MAGIC); + return Sample { + per_partition: sample.per_partition() as u64, + keys: sample.keys().iter().map(|key| *key as u64).collect(), + } + .encode(buf) + .map_err(|e| internal_datafusion_err!("Cannot encode a SampleExec: {e}")); } let Some(fragment) = node.downcast_ref::() else { return self.ballista.try_encode(node, buf, proto_converter); diff --git a/native/native-executors/src/frontend.rs b/native/native-executors/src/frontend.rs index 14822ed3278..60e7c8c2112 100644 --- a/native/native-executors/src/frontend.rs +++ b/native/native-executors/src/frontend.rs @@ -26,12 +26,14 @@ use std::sync::Arc; use std::time::{Duration, UNIX_EPOCH}; -use arrow_flight::flight_service_client::FlightServiceClient; use arrow_flight::flight_service_server::FlightService; use arrow_flight::{ Action, ActionType, Criteria, Empty, FlightData, FlightDescriptor, FlightEndpoint, FlightInfo, HandshakeRequest, HandshakeResponse, PollInfo, PutResult, SchemaResult, Ticket, }; +use ballista_core::client::BallistaClient; +use ballista_core::config::BallistaConfig; +use ballista_core::error::BallistaError; use ballista_core::extension::SessionConfigExt; use ballista_core::serde::protobuf::{job_status, PartitionLocation, SuccessfulJob}; use ballista_core::serde::scheduler::{Action as BallistaAction, ShuffleFileKind}; @@ -43,13 +45,13 @@ use datafusion::prelude::SessionConfig; use datafusion_comet_proto::spark_native_executors::QueryRequest; use datafusion_proto::protobuf::{LogicalPlanNode, PhysicalPlanNode}; use futures::stream::BoxStream; -use futures::TryStreamExt; +use futures::{StreamExt, TryStreamExt}; use log::info; use prost::Message; use prost_types::Timestamp; use tonic::{Request, Response, Status, Streaming}; -use crate::assembly::{build_plan_with_sampler, check_version}; +use crate::assembly::{build_plan, check_version}; /// The Ballista session every query runs in. Every query uses Ballista's default settings, and /// executors cache a runtime per session, so sharing one lets queries share those runtimes. @@ -66,12 +68,6 @@ impl CometFlightService { Self { scheduler } } - /// Runs `plan` to completion and returns its result: where its output partitions are, in - /// partition order, and when it ended. - async fn run(&self, plan: Arc) -> Result { - run(&self.scheduler, plan).await - } - /// When the executors delete the results of a query that ended at `ended_at`, in milliseconds /// since the epoch. Ballista deletes them that long after the query ends, unless the scheduler /// keeps them. Ballista starts that wait only after it records the end, so they are still @@ -92,49 +88,70 @@ impl CometFlightService { } } -/// Runs `plan` as a Ballista job on `scheduler` and reads the rows of each of its partitions back -/// from the executor that holds it, as a driver reads a query's result. +/// Runs `plan` as a Ballista job on `scheduler` and reads the rows of all its partitions back from +/// the executors that hold them, a few partitions at a time. async fn collect( scheduler: &SchedulerServer, plan: Arc, ) -> Result, Status> { let job = run(scheduler, plan).await?; - let mut batches = vec![]; - for location in &job.partition_location { - let endpoint = endpoint(location, None)?; - let uri = endpoint - .location - .first() - .map(|location| location.uri.replacen("grpc+tcp://", "http://", 1)) - .ok_or_else(|| Status::internal("A sample partition has no location"))?; - let channel = tonic::transport::Endpoint::from_shared(uri.clone()) - .map_err(|e| Status::internal(format!("Invalid executor location {uri}: {e}")))? - .connect() - .await - .map_err(|e| Status::internal(format!("Cannot connect to {uri}: {e}")))?; - let mut client = arrow_flight::FlightClient::new_from_inner( - FlightServiceClient::new(channel) - .max_decoding_message_size(usize::MAX) - .max_encoding_message_size(usize::MAX), - ); - let ticket = endpoint - .ticket - .ok_or_else(|| Status::internal("A sample partition has no ticket"))?; - let stream = client - .do_get(ticket) - .await - .map_err(|e| Status::internal(format!("Cannot read a sample from {uri}: {e}")))?; - batches.extend( - stream - .try_collect::>() - .await - .map_err(|e| Status::internal(format!("Cannot read a sample from {uri}: {e}")))?, - ); - } - Ok(batches) + let config = BallistaConfig::default(); + let batches: Vec> = futures::stream::iter(job.partition_location) + .map(|location| fetch(location, &config)) + .buffer_unordered(FETCH_CONCURRENCY) + .try_collect() + .await?; + Ok(batches.concat()) +} + +/// How many sample partitions the scheduler reads at once. +const FETCH_CONCURRENCY: usize = 16; + +/// Reads the rows of one partition from the executor that holds it, with Ballista's client. +async fn fetch( + location: PartitionLocation, + config: &BallistaConfig, +) -> Result, Status> { + let layout = location.layout(); + let (Some(executor), Some(partition)) = (location.executor_meta, location.partition_id) else { + return Err(Status::internal("A sample partition has no executor or ID")); + }; + let failed = |e: BallistaError| { + Status::internal(format!( + "Cannot read a sample from {}:{}: {e}", + executor.host, executor.port + )) + }; + let mut client = BallistaClient::try_new( + &executor.host, + executor.port as u16, + config.grpc_client_max_message_size(), + false, + None, + config.io_retries_times() as u8, + config.io_retry_wait_time_ms() as u64, + 0, + 0, + ) + .await + .map_err(failed)?; + let stream = client + .fetch_partition( + &executor.id, + &partition.into(), + location.file_id, + layout, + true, + ) + .await + .map_err(failed)?; + datafusion::physical_plan::common::collect(stream) + .await + .map_err(|e| failed(BallistaError::DataFusionError(Box::new(e)))) } -/// Runs `plan` as a Ballista job on `scheduler` and waits for it to end. +/// Runs `plan` as a Ballista job on `scheduler` to completion and returns its result: where its +/// output partitions are, in partition order, and when it ended. async fn run( scheduler: &SchedulerServer, plan: Arc, @@ -254,12 +271,12 @@ impl FlightService for CometFlightService { .block_on(collect(&scheduler, plan)) .map_err(|e| DataFusionError::External(Box::new(e))) }; - build_plan_with_sampler(query, &mut sampler) + build_plan(query, &mut sampler) }) .await .map_err(|e| Status::internal(e.to_string()))? .map_err(planning_status)?; - let job = self.run(plan).await?; + let job = run(&self.scheduler, plan).await?; let expires_at = self.results_expire_at(job.ended_at); let mut info = FlightInfo::new(); for location in &job.partition_location { diff --git a/native/native-executors/src/range.rs b/native/native-executors/src/range.rs index 25f993f34e4..668194907c3 100644 --- a/native/native-executors/src/range.rs +++ b/native/native-executors/src/range.rs @@ -18,20 +18,17 @@ //! Range exchanges: the split points of a range shuffle, found by sampling the exchange's input //! as Spark's `RangePartitioner` does. //! -//! The scheduler runs a sampling job over the stage that feeds a range exchange: a -//! [`SampleExec`] over each of its partitions, gathered to one partition. It then sorts the -//! sample by the exchange's sort keys and takes evenly spaced rows as the split points of a -//! [`RangePartitioning`](datafusion::physical_expr::Partitioning::Range), which Ballista plans as a -//! range shuffle. Like Spark's, the sampling job runs the producing stage once more. +//! The scheduler runs a sampling job over the stage that feeds a range exchange, a +//! [`SampleExec`] over each of its partitions, and reads every partition's sample back. It then +//! sorts the sample by the exchange's sort keys and takes evenly spaced rows as the split points +//! of a [`RangePartitioning`](datafusion::physical_expr::Partitioning::Range), which Ballista +//! plans as a range shuffle. Like Spark's, the sampling job runs the producing stage once more. use std::fmt; use std::sync::Arc; use datafusion::arrow::array::RecordBatch; -use datafusion::arrow::compute::{ - concat_batches, interleave_record_batch, lexsort_to_indices, take_record_batch, SortColumn, - SortOptions, -}; +use datafusion::arrow::compute::{concat_batches, interleave_record_batch, SortOptions}; use datafusion::arrow::datatypes::SchemaRef; use datafusion::arrow::row::{RowConverter, SortField}; use datafusion::common::tree_node::TreeNodeRecursion; @@ -156,24 +153,30 @@ impl ExecutionPlan for SampleExec { while let Some(batch) = input.next().await { let batch = batch?.project(&keys)?; // Where each row of the new reservoir comes from: (0, row) of the reservoir or - // (1, row) of the batch. - let mut indices: Vec<(usize, usize)> = - (0..reservoir.num_rows()).map(|row| (0, row)).collect(); - let mut changed = false; + // (1, row) of the batch. Built only once the batch changes the reservoir, which + // most batches of a large partition don't. + let mut indices: Option> = None; + let mut size = reservoir.num_rows(); for row in 0..batch.num_rows() { - if indices.len() < capacity { - indices.push((1, row)); - changed = true; + let slot = if size < capacity { + size += 1; + size - 1 } else { - let slot = rng.below(seen + 1) as usize; - if slot < capacity { + rng.below(seen + 1) as usize + }; + if slot < capacity { + let indices = indices.get_or_insert_with(|| { + (0..reservoir.num_rows()).map(|row| (0, row)).collect() + }); + if slot == indices.len() { + indices.push((1, row)); + } else { indices[slot] = (1, row); - changed = true; } } seen += 1; } - if changed { + if let Some(indices) = indices { reservoir = interleave_record_batch(&[&reservoir, &batch], &indices)?; } } @@ -226,16 +229,6 @@ pub fn split_points( if rows == 0 || partitions <= 1 { return Ok(vec![]); } - let columns: Vec = batch - .columns() - .iter() - .zip(options) - .map(|(values, options)| SortColumn { - values: Arc::clone(values), - options: Some(*options), - }) - .collect(); - let sorted = take_record_batch(&batch, &lexsort_to_indices(&columns, None)?)?; let converter = RowConverter::new( schema .fields() @@ -246,18 +239,22 @@ pub fn split_points( }) .collect(), )?; - let keys = converter.convert_columns(sorted.columns())?; + // The row format orders the keys as `options` say, so it both sorts the sample and tells + // equal neighbours apart. + let keys = converter.convert_columns(batch.columns())?; + let mut order: Vec = (0..rows).collect(); + order.sort_unstable_by(|a, b| keys.row(*a).cmp(&keys.row(*b))); let mut points = vec![]; let mut last = None; for i in 1..partitions { - let index = i * rows / partitions; + let index = order[i * rows / partitions]; let key = keys.row(index); // Split points must be strictly ordered. if last.is_some_and(|last| key <= keys.row(last)) { continue; } last = Some(index); - let values = sorted + let values = batch .columns() .iter() .map(|column| ScalarValue::try_from_array(column, index)) diff --git a/native/native-executors/tests/assembly.rs b/native/native-executors/tests/assembly.rs index 66c20ef784a..1f27b2a9007 100644 --- a/native/native-executors/tests/assembly.rs +++ b/native/native-executors/tests/assembly.rs @@ -20,7 +20,7 @@ mod common; -use comet_native_executors::assembly::{build_plan, build_plan_with_sampler}; +use comet_native_executors::assembly::build_plan; use common::*; use datafusion::arrow::datatypes::{DataType, Int32Type, Int64Type}; use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; @@ -38,10 +38,29 @@ use datafusion_comet_proto::spark_operator::{ use prost::Message; use std::sync::Arc; +/// Assembles `request`, running each sampling plan in this process. async fn assemble(request: QueryRequest) -> datafusion::common::Result> { - tokio::task::spawn_blocking(move || build_plan(request)) + assemble_counting_samples(request) .await - .unwrap() + .map(|(plan, _)| plan) +} + +/// Assembles `request` like [`assemble`], and also returns how many sampling plans ran. +async fn assemble_counting_samples( + request: QueryRequest, +) -> datafusion::common::Result<(Arc, usize)> { + let runtime = tokio::runtime::Handle::current(); + tokio::task::spawn_blocking(move || { + let mut samples = 0; + let mut sampler = |plan: Arc| { + samples += 1; + runtime.block_on(collect(plan, SessionContext::new().task_ctx())) + }; + let plan = build_plan(request, &mut sampler)?; + Ok((plan, samples)) + }) + .await + .unwrap() } /// Runs `request` and returns its single column, sorted. @@ -78,34 +97,17 @@ async fn single_partition_exchange_collects_every_row() { } #[tokio::test(flavor = "multi_thread")] -async fn range_exchange_is_read_as_one_partition() { - // The driver asks for the exchange's partition count. +async fn range_exchange_without_column_keys_is_read_as_one_partition() { + // The driver asks for the exchange's partition count, but sends no sort key to sample. let request = query(vec![ range_stage(100, 4), - reading_stage(0, range_partitioning(4), 4), + reading_stage(0, range_partitioning(4, vec![]), 4), ]); let plan = assemble(request.clone()).await.unwrap(); assert_eq!(plan.output_partitioning().partition_count(), 1); assert_eq!(run(request).await, (0..100).collect::>()); } -/// Assembles `request` with a sampler that runs each sampling plan in this process, and returns -/// the plan and how many sampling plans ran. -async fn assemble_sampling(request: QueryRequest) -> (Arc, usize) { - let runtime = tokio::runtime::Handle::current(); - tokio::task::spawn_blocking(move || { - let mut samples = 0; - let mut sampler = |plan: Arc| { - samples += 1; - runtime.block_on(collect(plan, SessionContext::new().task_ctx())) - }; - let plan = build_plan_with_sampler(request, &mut sampler).unwrap(); - (plan, samples) - }) - .await - .unwrap() -} - /// The exchange a reading stage reads: its second child, after the partition descriptor. fn exchange_of(plan: &Arc) -> Arc { Arc::clone(plan.children()[1]) @@ -115,9 +117,9 @@ fn exchange_of(plan: &Arc) -> Arc { async fn range_exchange_is_range_partitioned_by_a_sample() { let request = query(vec![ range_stage(1000, 4), - reading_stage(0, range_on_first_column_descending(4), 4), + reading_stage(0, range_partitioning(4, vec![first_column_descending()]), 4), ]); - let (plan, samples) = assemble_sampling(request).await; + let (plan, samples) = assemble_counting_samples(request).await.unwrap(); assert_eq!(samples, 1); assert_eq!(plan.output_partitioning().partition_count(), 4); let exchange = exchange_of(&plan); @@ -141,9 +143,9 @@ async fn range_exchange_on_a_float_key_is_read_as_one_partition() { producer.output_schema = vec![field(double_type())]; let request = query(vec![ producer, - reading_stage(0, range_on_first_column_descending(4), 4), + reading_stage(0, range_partitioning(4, vec![first_column_descending()]), 4), ]); - let (plan, samples) = assemble_sampling(request).await; + let (plan, samples) = assemble_counting_samples(request).await.unwrap(); assert_eq!(samples, 0); assert!(exchange_of(&plan).is::()); } diff --git a/native/native-executors/tests/cluster.rs b/native/native-executors/tests/cluster.rs index aa7da7f2609..e38d5190ff8 100644 --- a/native/native-executors/tests/cluster.rs +++ b/native/native-executors/tests/cluster.rs @@ -197,7 +197,7 @@ async fn runs_a_query_across_a_range_exchange() { let cluster = Cluster::start(); let request = query(vec![ range_stage(1000, 4), - reading_stage(0, range_partitioning(4), 4), + reading_stage(0, range_partitioning(4, vec![]), 4), ]); let partitions = cluster.run(&request).await; assert_eq!(partitions.len(), 1); @@ -214,7 +214,7 @@ async fn range_partitions_a_range_exchange_by_sampled_split_points() { let cluster = Cluster::start(); let request = query(vec![ range_stage(1000, 4), - reading_stage(0, range_on_first_column_descending(4), 4), + reading_stage(0, range_partitioning(4, vec![first_column_descending()]), 4), ]); let partitions = cluster.run(&request).await; assert_eq!(partitions.len(), 4); diff --git a/native/native-executors/tests/common/mod.rs b/native/native-executors/tests/common/mod.rs index 79fb3ec9e47..112a509581f 100644 --- a/native/native-executors/tests/common/mod.rs +++ b/native/native-executors/tests/common/mod.rs @@ -146,32 +146,26 @@ pub fn single_partition() -> Partitioning { } } -/// A range exchange as a driver sends it: its sort keys don't matter here, and it has no bounds. -pub fn range_partitioning(num_partitions: i32) -> Partitioning { +/// A range exchange as a driver sends it, with no bounds: sorting by `sort_orders`. +pub fn range_partitioning(num_partitions: i32, sort_orders: Vec) -> Partitioning { Partitioning { partitioning_struct: Some(PartitioningStruct::RangePartition(RangePartition { + sort_orders, num_partitions, ..Default::default() })), } } -/// A range exchange as a driver sends it, sorting by the first column, descending. -pub fn range_on_first_column_descending(num_partitions: i32) -> Partitioning { - let order = Expr { +/// The first column, descending, as a range exchange's sort key. +pub fn first_column_descending() -> Expr { + Expr { expr_struct: Some(ExprStruct::SortOrder(Box::new(SortOrder { child: Some(Box::new(bound(0, long_type()))), direction: SortDirection::Descending as i32, null_ordering: NullOrdering::NullsLast as i32, }))), ..Default::default() - }; - Partitioning { - partitioning_struct: Some(PartitioningStruct::RangePartition(RangePartition { - sort_orders: vec![order], - num_partitions, - ..Default::default() - })), } } diff --git a/native/proto/src/proto/native_executors.proto b/native/proto/src/proto/native_executors.proto index a324dc690a1..9c3ae718c59 100644 --- a/native/proto/src/proto/native_executors.proto +++ b/native/proto/src/proto/native_executors.proto @@ -72,6 +72,13 @@ message Fragment { map conf = 4; } +// How a scheduler sends the sampling operator of a range exchange to its executors: a +// reservoir sample of up to `per_partition` rows of each partition, keeping the columns `keys`. +message Sample { + uint64 per_partition = 1; + repeated uint64 keys = 2; +} + message StageInput { // Index into QueryRequest.stages of the stage producing this input. uint32 producer = 1; diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala index 3c2489c52e4..07b70a956c8 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala @@ -46,7 +46,7 @@ import com.google.protobuf.Message import org.apache.comet.{CometConf, CometExecIterator, CometExplainInfo, CometRuntimeException, NativeExecutorsClient} import org.apache.comet.serde.{ExprOuterClass, PartitioningOuterClass, QueryPlanSerde} import org.apache.comet.serde.Config.ConfigMap -import org.apache.comet.serde.NativeExecutors.{QueryRequest, ResultEndpoints, Stage, StageInput} +import org.apache.comet.serde.NativeExecutors.{QueryRequest, ResultEndpoint, ResultEndpoints, Stage, StageInput} import org.apache.comet.serde.OperatorOuterClass.Operator import org.apache.comet.serde.operator.schema2Proto import org.apache.comet.vector.NativeUtil @@ -83,20 +83,17 @@ case class CometNativeExecutorsExec( subqueryResults.collect { case sub: ScalarSubquery => sub }) val endpoints = ResultEndpoints.parseFrom(new NativeExecutorsClient().submit(schedulerUrl, request)) - val partitions = endpoints.getEndpointsList.asScala.map { endpoint => - (endpoint.getLocation, endpoint.getTicket.toByteArray) - } - new CometNativeExecutorsRDD(sparkContext, partitions.toSeq, output.length) + new CometNativeExecutorsRDD( + sparkContext, + endpoints.getEndpointsList.asScala.toSeq, + output.length) } override def stringArgs: Iterator[Any] = Iterator(output, s"${query.length} bytes", schedulerUrl) } -private class CometNativeExecutorsPartition( - override val index: Int, - val location: String, - val ticket: Array[Byte]) +private class CometNativeExecutorsPartition(override val index: Int, val endpoint: ResultEndpoint) extends Partition /** @@ -105,28 +102,30 @@ private class CometNativeExecutorsPartition( */ private class CometNativeExecutorsRDD( sc: SparkContext, - endpoints: Seq[(String, Array[Byte])], + endpoints: Seq[ResultEndpoint], numColumns: Int) extends RDD[ColumnarBatch](sc, Nil) { override protected def getPartitions: Array[Partition] = - endpoints.zipWithIndex.map { case ((location, ticket), index) => - new CometNativeExecutorsPartition(index, location, ticket): Partition + endpoints.zipWithIndex.map { case (endpoint, index) => + new CometNativeExecutorsPartition(index, endpoint): Partition }.toArray override def compute(split: Partition, context: TaskContext): Iterator[ColumnarBatch] = { val partition = split.asInstanceOf[CometNativeExecutorsPartition] val client = new NativeExecutorsClient() val nativeUtil = new NativeUtil() - val handle = client.openPartition(partition.location, partition.ticket) + val handle = + client.openPartition( + partition.endpoint.getLocation, + partition.endpoint.getTicket.toByteArray) val batches = new Iterator[ColumnarBatch] with AutoCloseable { private var current: Option[ColumnarBatch] = None private var pending: Option[ColumnarBatch] = None - private var finished = false private var closed = false override def hasNext: Boolean = { - if (pending.isEmpty && !finished) { + if (pending.isEmpty && !closed) { // The consumer has moved past the batch it last got. current.foreach(_.close()) current = None @@ -134,7 +133,6 @@ private class CometNativeExecutorsRDD( numColumns, (arrays, schemas) => client.nextBatch(handle, arrays, schemas)) if (pending.isEmpty) { - finished = true close() } } @@ -174,7 +172,8 @@ object CometNativeExecutors { /** * Messages of native plans that call back into the JVM, which a cluster doesn't have. A scalar * subquery's `Subquery` is not one of them: the driver runs the subquery and puts its result in - * the plan before submitting it. + * the plan before submitting it. The native planner could refuse them all instead + * (https://github.com/apache/datafusion-comet/issues/7375). */ private val jvmCallbacks = Set("JvmScalarUdf", "ArrowPythonUdf") @@ -252,7 +251,8 @@ object CometNativeExecutors { /** * `query` with each scalar subquery's `Subquery` expression replaced by a literal of its - * result. The subqueries must have run. + * result. The subqueries must have run. Sending the results alongside the plan would spare + * rewriting it (https://github.com/apache/datafusion-comet/issues/7374). */ def withSubqueryResults(query: Array[Byte], subqueries: Seq[ScalarSubquery]): Array[Byte] = { if (subqueries.isEmpty) { @@ -267,55 +267,82 @@ object CometNativeExecutors { s"Comet Native Executors cannot pass the result of $sub to the cluster")) sub.exprId.id -> expr }.toMap + val replace: PartialFunction[Message, Message] = { + case expr: ExprOuterClass.Expr if expr.hasSubquery => + literals.getOrElse( + expr.getSubquery.getId, + throw new CometRuntimeException( + s"Comet Native Executors found no result for subquery ${expr.getSubquery.getId}")) + } val request = QueryRequest.parseFrom(query).toBuilder (0 until request.getStagesCount).foreach { i => - val stage = request.getStages(i).toBuilder - stage.setPlan(replaceSubqueries(stage.getPlan, literals).asInstanceOf[Operator]) - (0 until stage.getPartitionPlansCount).foreach { p => - val plan = Operator.parseFrom(stage.getPartitionPlans(p)) - stage.setPartitionPlans( - p, - ByteString.copyFrom(replaceSubqueries(plan, literals).toByteArray)) + val stage = request.getStages(i) + val plan = transformMessages(stage.getPlan)(replace) + // A partition's plan is the stage's with its files added, so it holds a subquery only + // when the stage's plan does. + if (plan ne stage.getPlan) { + val replaced = stage.toBuilder.setPlan(plan.asInstanceOf[Operator]) + (0 until stage.getPartitionPlansCount).foreach { p => + val partitionPlan = Operator.parseFrom(stage.getPartitionPlans(p)) + replaced.setPartitionPlans(p, transformMessages(partitionPlan)(replace).toByteString) + } + request.setStages(i, replaced) } - request.setStages(i, stage) } request.build().toByteArray } - private def replaceSubqueries( - message: Message, - literals: Map[Long, ExprOuterClass.Expr]): Message = message match { - case expr: ExprOuterClass.Expr if expr.hasSubquery => - literals.getOrElse( - expr.getSubquery.getId, - throw new CometRuntimeException( - s"Comet Native Executors found no result for subquery ${expr.getSubquery.getId}")) - case _ => - val builder = message.toBuilder - var changed = false - message.getAllFields.asScala.foreach { case (field, value) => - if (field.getJavaType == FieldDescriptor.JavaType.MESSAGE) { - if (field.isRepeated) { - value.asInstanceOf[java.util.List[Message]].asScala.zipWithIndex.foreach { - case (element, index) => - val replaced = replaceSubqueries(element, literals) - if (replaced ne element) { - builder.setRepeatedField(field, index, replaced) - changed = true - } - } - } else { - val element = value.asInstanceOf[Message] - val replaced = replaceSubqueries(element, literals) - if (replaced ne element) { - builder.setField(field, replaced) - changed = true - } + /** + * The messages directly inside `message`, each with how to put a replacement for it in a + * builder of `message`. + */ + private def childMessages( + message: Message): Iterator[(Message, (Message.Builder, Message) => Unit)] = + message.getAllFields.asScala.iterator + .filter { case (field, _) => field.getJavaType == FieldDescriptor.JavaType.MESSAGE } + .flatMap { case (field, value) => + if (field.isRepeated) { + value.asInstanceOf[java.util.List[Message]].asScala.iterator.zipWithIndex.map { + case (child, index) => + ( + child, + (builder: Message.Builder, m: Message) => { + builder.setRepeatedField(field, index, m) + () + }) } + } else { + Iterator( + ( + value.asInstanceOf[Message], + (builder: Message.Builder, m: Message) => { + builder.setField(field, m) + () + })) } } - if (changed) builder.build() else message - } + + /** + * `message` with every message inside it that `pf` matches replaced by what `pf` returns. A + * message nothing inside changed is returned as is. + */ + private def transformMessages(message: Message)( + pf: PartialFunction[Message, Message]): Message = + pf.applyOrElse( + message, + (m: Message) => { + var builder: Message.Builder = null + childMessages(m).foreach { case (child, set) => + val replaced = transformMessages(child)(pf) + if (replaced ne child) { + if (builder == null) { + builder = m.toBuilder + } + set(builder, replaced) + } + } + if (builder == null) m else builder.build() + }) /** The configs a Spark task would pass to native code with a plan. */ private def nativeConf(): java.util.Map[String, String] = { @@ -326,6 +353,11 @@ object CometNativeExecutors { conf } + private val singlePartition = PartitioningOuterClass.Partitioning + .newBuilder() + .setSinglePartition(PartitioningOuterClass.SinglePartition.newBuilder()) + .build() + private class StageGraphBuilder(query: SparkPlan) { val stages: ArrayBuffer[Stage] = ArrayBuffer.empty private val stageIndexes = new IdentityHashMap[SparkPlan, Integer]() @@ -343,17 +375,8 @@ object CometNativeExecutors { } val inputs = inputsOf(block).map { case exchange: CometShuffleExchangeExec => - StageInput - .newBuilder() - .setProducer(stageOf(exchange.child)) - .setPartitioning(partitioningOf(exchange)) - .build() - case broadcast => - StageInput - .newBuilder() - .setProducer(stageOf(broadcast.children.head)) - .setBroadcast(true) - .build() + stageInput(exchange.child)(_.setPartitioning(partitioningOf(exchange))) + case broadcast => stageInput(broadcast.children.head)(_.setBroadcast(true)) } val blockPlan = Operator.parseFrom(block.serializedPlanOpt.plan.get) @@ -373,27 +396,27 @@ object CometNativeExecutors { val numPartitions = if (partitionPlans.nonEmpty) partitionPlans.length else block.outputPartitioning.numPartitions - addStage(plan, blockPlan, inputs, partitionPlans, numPartitions, block.output) + addStage(plan, blockPlan, inputs, partitionPlans, numPartitions) } + /** An input of a stage that reads the stage running `producer`, as `configure` says. */ + private def stageInput(producer: SparkPlan)( + configure: StageInput.Builder => StageInput.Builder): StageInput = + configure(StageInput.newBuilder().setProducer(stageOf(producer))).build() + /** * A stage for `take` that reads its child's whole output as one partition and keeps the first - * rows in order, as Comet's final top-K plan does after its shuffle to one partition. + * rows in order, as Comet's final top-K plan does after its shuffle to one partition. A top-K + * of each partition first would send fewer rows through the exchange + * (https://github.com/apache/datafusion-comet/issues/7365). */ private def takeOrderedStage(take: CometTakeOrderedAndProjectExec): Int = { - val input = StageInput - .newBuilder() - .setProducer(stageOf(take.child)) - .setPartitioning( - PartitioningOuterClass.Partitioning - .newBuilder() - .setSinglePartition(PartitioningOuterClass.SinglePartition.newBuilder())) - .build() + val input = stageInput(take.child)(_.setPartitioning(singlePartition)) // More than one input partition, so that the plan sorts across all of them. val blockPlan = take .finalNativePlan(inputPartitions = 2) .getOrElse(fail(take, "its top-K plan cannot run natively")) - addStage(take, blockPlan, Seq(input), Seq.empty, 1, take.output) + addStage(take, blockPlan, Seq(input), Seq.empty, 1) } private def addStage( @@ -401,15 +424,14 @@ object CometNativeExecutors { blockPlan: Operator, inputs: Seq[StageInput], partitionPlans: Seq[ByteString], - numPartitions: Int, - output: Seq[Attribute]): Int = { + numPartitions: Int): Int = { findJvmCallback(blockPlan).foreach { name => fail(plan, s"its native plan calls back into the JVM ($name)") } // Declared as Comet's shuffle writer declares its input, since the cluster casts the // block's output to it as that writer does. - val outputSchema = - schema2Proto(output.map(a => StructField(a.name, a.dataType, a.nullable, a.metadata))) + val outputSchema = schema2Proto( + plan.output.map(a => StructField(a.name, a.dataType, a.nullable, a.metadata))) stages += Stage .newBuilder() .setPlan(blockPlan) @@ -450,7 +472,7 @@ object CometNativeExecutors { private def partitioningOf( exchange: CometShuffleExchangeExec): PartitioningOuterClass.Partitioning = { - val partitioning = PartitioningOuterClass.Partitioning.newBuilder() + def partitioning = PartitioningOuterClass.Partitioning.newBuilder() exchange.outputPartitioning match { case HashPartitioning(expressions, numPartitions) => val keys = expressions.map { key => @@ -458,11 +480,13 @@ object CometNativeExecutors { .exprToProto(key, exchange.child.output) .getOrElse(fail(exchange, s"its partitioning key $key cannot run natively")) } - partitioning.setHashPartition( - PartitioningOuterClass.HashPartition - .newBuilder() - .addAllHashExpression(keys.asJava) - .setNumPartitions(numPartitions)) + partitioning + .setHashPartition( + PartitioningOuterClass.HashPartition + .newBuilder() + .addAllHashExpression(keys.asJava) + .setNumPartitions(numPartitions)) + .build() case RangePartitioning(ordering, numPartitions) => // Without bounds: the cluster decides how to range-partition. val sortOrders = ordering.map { order => @@ -470,17 +494,17 @@ object CometNativeExecutors { .exprToProto(order, exchange.child.output) .getOrElse(fail(exchange, s"its sort key $order cannot run natively")) } - partitioning.setRangePartition( - PartitioningOuterClass.RangePartition - .newBuilder() - .addAllSortOrders(sortOrders.asJava) - .setNumPartitions(numPartitions)) - case SinglePartition => - partitioning.setSinglePartition(PartitioningOuterClass.SinglePartition.newBuilder()) + partitioning + .setRangePartition( + PartitioningOuterClass.RangePartition + .newBuilder() + .addAllSortOrders(sortOrders.asJava) + .setNumPartitions(numPartitions)) + .build() + case SinglePartition => singlePartition case other => fail(exchange, s"Comet Native Executors do not support $other exchanges yet") } - partitioning.build() } private def fail(node: SparkPlan, reason: String): Nothing = { @@ -498,22 +522,11 @@ object CometNativeExecutors { } } + /** `message` and every message inside it, depth first. */ + private def messagesIn(message: Message): Iterator[Message] = + Iterator(message) ++ childMessages(message).flatMap { case (child, _) => messagesIn(child) } + /** The first message in `message` that calls back into the JVM, if any. */ - private def findJvmCallback(message: Message): Option[String] = { - val name = message.getDescriptorForType.getName - if (jvmCallbacks.contains(name)) { - Some(name) - } else { - val found = message.getAllFields.asScala.iterator.flatMap { case (field, value) => - if (field.getJavaType != FieldDescriptor.JavaType.MESSAGE) { - Iterator.empty - } else if (field.isRepeated) { - value.asInstanceOf[java.util.List[Message]].asScala.iterator.flatMap(findJvmCallback) - } else { - findJvmCallback(value.asInstanceOf[Message]).iterator - } - } - if (found.hasNext) Some(found.next()) else None - } - } + private def findJvmCallback(message: Message): Option[String] = + messagesIn(message).map(_.getDescriptorForType.getName).find(jvmCallbacks.contains) } diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala index ef3a585fd1f..6e321279aa6 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala @@ -27,7 +27,7 @@ import java.util.concurrent.TimeUnit import scala.util.Try import org.apache.spark.SparkConf -import org.apache.spark.sql.{CometTestBase, DataFrame} +import org.apache.spark.sql.{CometTestBase, DataFrame, Row} import org.apache.spark.sql.comet.CometNativeExecutorsExec import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.functions.{col, count, max, sum} @@ -142,15 +142,26 @@ class CometNativeExecutorsSuite extends CometTestBase with AdaptiveSparkPlanHelp CometConf.COMET_NATIVE_EXECUTORS_ENABLED.key -> "true", CometConf.COMET_NATIVE_EXECUTORS_SCHEDULER_URL.key -> schedulerUrl)(body) - /** Checks that the cluster runs `query` and returns what Spark returns. */ - private def checkOnCluster(query: => DataFrame): Unit = { - val expected = query.collect().toSeq - onCluster { - val df = query + /** + * Checks that the cluster runs `query` and returns what Spark returns, in order if `ordered`. + */ + private def checkOnCluster(query: => DataFrame, ordered: Boolean = false): Unit = + checkClusterReturns(query, query.collect().toSeq, ordered) + + /** Checks that the cluster runs `query` and returns `expected`, in order if `ordered`. */ + private def checkClusterReturns( + query: => DataFrame, + expected: Seq[Row], + ordered: Boolean): Unit = onCluster { + // A new Dataset, since a Dataset keeps the plan it was first run with. + val df = query + if (ordered) { + assert(df.collect().toSeq == expected, "the cluster returned other rows or another order") + } else { checkAnswer(df, expected) - val plan = df.queryExecution.executedPlan - assert(collect(plan) { case exec: CometNativeExecutorsExec => exec }.size == 1, plan) } + val plan = df.queryExecution.executedPlan + assert(collect(plan) { case exec: CometNativeExecutorsExec => exec }.size == 1, plan) } /** A Parquet table with several files, read as several partitions. */ @@ -265,12 +276,9 @@ class CometNativeExecutorsSuite extends CometTestBase with AdaptiveSparkPlanHelp nativeExecutorsTest("TPC-H q1") { withLineitem { - // A new Dataset each time, since a Dataset keeps the plan it was first run with. - def q1 = sql(s"$q1Aggregate\norder by\n l_returnflag,\n l_linestatus") - checkOnCluster(q1) - // checkAnswer ignores the order of the rows. - val expected = q1.collect().toSeq - onCluster(assert(q1.collect().toSeq == expected)) + checkOnCluster( + sql(s"$q1Aggregate\norder by\n l_returnflag,\n l_linestatus"), + ordered = true) } } @@ -319,17 +327,7 @@ class CometNativeExecutorsSuite extends CometTestBase with AdaptiveSparkPlanHelp SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") { sql(query).collect().toSeq } - onCluster { - val df = sql(query) - val actual = df.collect().toSeq - val plan = df.queryExecution.executedPlan - assert(collect(plan) { case exec: CometNativeExecutorsExec => exec }.nonEmpty, plan) - if (query.toLowerCase.contains("order by")) { - assert(actual == expected, s"q$q returned rows in another order or other rows") - } else { - checkAnswer(df, expected) - } - } + checkClusterReturns(sql(query), expected, query.toLowerCase.contains("order by")) } finally { teardown.foreach(sql) } From a3993563df2d238db82f09ca782cddc2a886b3a7 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 31 Oct 2026 13:00:51 -0600 Subject: [PATCH 17/36] feat: run Comet Native Executor queries in Ballista's pipelined mode COMET_SCHEDULER_BALLISTA_CONFIG sets Ballista settings, such as ballista.execution.mode=pipelined, for every query. A pipelined job's final stage streams each result partition to one reader, once, and can't finish until it is read, while Spark may read a result partition more than once. So the scheduler reads a pipelined query's results itself, gathered into one partition, and serves them from its own Flight service for as long as a blocking query's results are kept. The driver reads an endpoint without a location from the scheduler. A pipelined job can't range-partition, so the scheduler doesn't sample range exchanges in that mode. The workspace now builds on comet-native at 8ab06362a, which merges Ballista's main branch with its pipelined execution mode. --- .../src/execution/native_executors/client.rs | 4 +- native/native-executors/Cargo.lock | 12 +- .../src/bin/comet-scheduler.rs | 21 +- native/native-executors/src/frontend.rs | 288 ++++++++++++++++-- 4 files changed, 289 insertions(+), 36 deletions(-) diff --git a/native/core/src/execution/native_executors/client.rs b/native/core/src/execution/native_executors/client.rs index c78d1ca09bd..d5360f54fe4 100644 --- a/native/core/src/execution/native_executors/client.rs +++ b/native/core/src/execution/native_executors/client.rs @@ -83,11 +83,11 @@ async fn submit(scheduler_url: &str, mut query: Vec) -> CometResult Result<(), Box> { config.finished_job_data_clean_up_interval_seconds = seconds; } let flight_port: u16 = env_var("COMET_SCHEDULER_FLIGHT_PORT")?.unwrap_or(50060); + let ballista_settings = + parse_settings(&env_var::("COMET_SCHEDULER_BALLISTA_CONFIG")?.unwrap_or_default())?; let grpc_addr: SocketAddr = format!("{}:{}", config.bind_host, config.bind_port).parse()?; let flight_addr = SocketAddr::new(grpc_addr.ip(), flight_port); let config = Arc::new(config); @@ -68,7 +72,7 @@ async fn main() -> Result<(), Box> { let grpc = tokio::spawn(start_grpc_service(grpc_addr, scheduler.clone())); let flight = tonic::transport::Server::builder() .add_service( - FlightServiceServer::new(CometFlightService::new(scheduler)) + FlightServiceServer::new(CometFlightService::try_new(scheduler, &ballista_settings)?) .max_decoding_message_size(usize::MAX) .max_encoding_message_size(usize::MAX), ) @@ -79,3 +83,18 @@ async fn main() -> Result<(), Box> { } Ok(()) } + +/// `key=value` pairs separated by commas. +fn parse_settings(settings: &str) -> Result, String> { + settings + .split(',') + .map(str::trim) + .filter(|setting| !setting.is_empty()) + .map(|setting| { + setting + .split_once('=') + .map(|(key, value)| (key.trim().to_string(), value.trim().to_string())) + .ok_or_else(|| format!("Invalid Ballista setting {setting:?}: expected key=value")) + }) + .collect() +} diff --git a/native/native-executors/src/frontend.rs b/native/native-executors/src/frontend.rs index 60e7c8c2112..a2a7940789d 100644 --- a/native/native-executors/src/frontend.rs +++ b/native/native-executors/src/frontend.rs @@ -23,24 +23,30 @@ //! also says when the partition goes: Ballista deletes a query's results a set time after the //! query ends, whether or not they have been read (see the comet-scheduler binary). -use std::sync::Arc; -use std::time::{Duration, UNIX_EPOCH}; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use arrow_flight::encode::FlightDataEncoderBuilder; use arrow_flight::flight_service_server::FlightService; use arrow_flight::{ Action, ActionType, Criteria, Empty, FlightData, FlightDescriptor, FlightEndpoint, FlightInfo, HandshakeRequest, HandshakeResponse, PollInfo, PutResult, SchemaResult, Ticket, }; use ballista_core::client::BallistaClient; -use ballista_core::config::BallistaConfig; +use ballista_core::config::{BallistaConfig, ExecutionMode}; use ballista_core::error::BallistaError; use ballista_core::extension::SessionConfigExt; -use ballista_core::serde::protobuf::{job_status, PartitionLocation, SuccessfulJob}; +use ballista_core::serde::protobuf::{ + job_status, JobStatus, PartitionLocation, RunningJob, SuccessfulJob, +}; use ballista_core::serde::scheduler::{Action as BallistaAction, ShuffleFileKind}; use ballista_scheduler::scheduler_server::SchedulerServer; use datafusion::arrow::array::RecordBatch; +use datafusion::arrow::datatypes::SchemaRef; use datafusion::common::DataFusionError; -use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; +use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties}; use datafusion::prelude::SessionConfig; use datafusion_comet_proto::spark_native_executors::QueryRequest; use datafusion_proto::protobuf::{LogicalPlanNode, PhysicalPlanNode}; @@ -61,17 +67,68 @@ const SESSION_ID: &str = "comet"; /// wraps. pub struct CometFlightService { scheduler: SchedulerServer, + /// The results of pipelined queries, by job id, until they expire. + stored_results: Arc>>>, + /// The session every query runs in, with the cluster's Ballista settings. + session: SessionConfig, } impl CometFlightService { - pub fn new(scheduler: SchedulerServer) -> Self { - Self { scheduler } + /// A service that runs each query in a session with Ballista's defaults overridden by + /// `ballista_settings`, such as `ballista.execution.mode=pipelined`. + pub fn try_new( + scheduler: SchedulerServer, + ballista_settings: &[(String, String)], + ) -> Result { + let session = ballista_settings.iter().fold( + SessionConfig::new_with_ballista(), + |session, (key, value)| session.set_str(key, value), + ); + // Check the settings now rather than when the first query runs. + session + .ballista_config() + .execution_mode() + .map_err(|e| format!("Invalid Ballista settings: {e}"))?; + Ok(Self { + scheduler, + stored_results: Arc::default(), + session, + }) + } + + /// Whether queries run in Ballista's pipelined mode, which streams batches between stages + /// instead of writing shuffle files, and can't range-partition. + fn pipelined(&self) -> bool { + matches!( + self.session.ballista_config().execution_mode(), + Ok(ExecutionMode::Pipelined) + ) } /// When the executors delete the results of a query that ended at `ended_at`, in milliseconds /// since the epoch. Ballista deletes them that long after the query ends, unless the scheduler /// keeps them. Ballista starts that wait only after it records the end, so they are still /// there until this time. + /// Keeps a pipelined query's result until it expires, as long as a blocking job's. + fn store(&self, job_id: String, result: StoredResult) { + self.stored_results + .lock() + .unwrap() + .insert(job_id.clone(), Arc::new(result)); + let retention = self + .scheduler + .state + .config + .finished_job_data_clean_up_interval_seconds; + if retention > 0 { + let stored = Arc::clone(&self.stored_results); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(retention)).await; + stored.lock().unwrap().remove(&job_id); + }); + } + } + fn results_expire_at(&self, ended_at: u64) -> Option { let retention = self .scheduler @@ -92,9 +149,10 @@ impl CometFlightService { /// the executors that hold them, a few partitions at a time. async fn collect( scheduler: &SchedulerServer, + session: &SessionConfig, plan: Arc, ) -> Result, Status> { - let job = run(scheduler, plan).await?; + let job = run(scheduler, session, plan).await?; let config = BallistaConfig::default(); let batches: Vec> = futures::stream::iter(job.partition_location) .map(|location| fetch(location, &config)) @@ -154,20 +212,10 @@ async fn fetch( /// output partitions are, in partition order, and when it ended. async fn run( scheduler: &SchedulerServer, + session: &SessionConfig, plan: Arc, ) -> Result { - let ctx = scheduler - .state - .session_manager - .create_or_update_session(SESSION_ID, &SessionConfig::new_with_ballista()) - .await - .map_err(|e| Status::internal(format!("Cannot create a session: {e}")))?; - let (sender, mut receiver) = tokio::sync::mpsc::channel(16); - let job_id = scheduler - .submit_physical_plan("comet", ctx, plan, Some(sender)) - .await - .map_err(|e| Status::internal(format!("Cannot submit the query: {e}")))?; - info!("Submitted query as job {job_id}"); + let (job_id, mut receiver) = submit(scheduler, session, plan).await?; while let Some(status) = receiver.recv().await { match status.status { Some(job_status::Status::Successful(mut job)) => { @@ -191,6 +239,142 @@ async fn run( ))) } +/// Submits `plan` as a Ballista job on `scheduler`, and returns its id and the channel its status +/// changes arrive on. +async fn submit( + scheduler: &SchedulerServer, + session: &SessionConfig, + plan: Arc, +) -> Result<(String, tokio::sync::mpsc::Receiver), Status> { + let ctx = scheduler + .state + .session_manager + .create_or_update_session(SESSION_ID, session) + .await + .map_err(|e| Status::internal(format!("Cannot create a session: {e}")))?; + let (sender, receiver) = tokio::sync::mpsc::channel(16); + let job_id = scheduler + .submit_physical_plan("comet", ctx, plan, Some(sender)) + .await + .map_err(|e| Status::internal(format!("Cannot submit the query: {e}")))?; + info!("Submitted query as job {job_id}"); + Ok((job_id.to_string(), receiver)) +} + +/// Runs `plan` as a pipelined Ballista job and reads its results into the scheduler. +/// +/// A pipelined job's final stage streams each output partition to one reader, once, and can't +/// finish until it has been read. A Spark driver may read a result partition more than once, +/// since Spark can compute a partition of an RDD again, so the scheduler reads the streams itself +/// and serves them from its own Flight service for as long as a blocking job's results are kept. +/// +/// The result is gathered into one partition first. A pipelined query's order lives in one +/// partition anyway, since a pipelined job reads a range exchange as one partition. +async fn run_pipelined( + scheduler: &SchedulerServer, + session: &SessionConfig, + plan: Arc, +) -> Result<(String, StoredResult), Status> { + let plan: Arc = if plan.output_partitioning().partition_count() > 1 { + Arc::new(CoalescePartitionsExec::new(plan)) + } else { + plan + }; + let schema = plan.schema(); + let config = session.ballista_config(); + let (job_id, mut receiver) = submit(scheduler, session, plan).await?; + let failed = + |job_id: &str, error: &str| Status::internal(format!("Job {job_id} failed: {error}")); + let locations = loop { + let Some(status) = receiver.recv().await else { + return Err(Status::internal(format!( + "Job {job_id} ended without reporting where its results are" + ))); + }; + match status.status { + Some(job_status::Status::Running(RunningJob { + pipelined_output_locations, + .. + })) + | Some(job_status::Status::Successful(SuccessfulJob { + pipelined_output_locations, + .. + })) if !pipelined_output_locations.is_empty() => break pipelined_output_locations, + Some(job_status::Status::Failed(job)) => return Err(failed(&job_id, &job.error)), + _ => {} + } + }; + // Every partition at once: the final stage can't finish until all of them are read. + let partitions = futures::future::try_join_all( + locations + .into_iter() + .map(|location| fetch_pipelined(location, &config)), + ) + .await; + // The job's own failure explains a failed read better than the read does. + while let Some(status) = receiver.recv().await { + match status.status { + Some(job_status::Status::Successful(_)) => break, + Some(job_status::Status::Failed(job)) => return Err(failed(&job_id, &job.error)), + _ => {} + } + } + Ok(( + job_id, + StoredResult { + schema, + partitions: partitions?, + }, + )) +} + +/// Reads one result partition of a pipelined job from the executor that streams it. +async fn fetch_pipelined( + location: ballista_core::serde::protobuf::PipelinedPartitionLocation, + config: &BallistaConfig, +) -> Result, Status> { + let location: ballista_core::serde::scheduler::PipelinedPartitionLocation = location + .try_into() + .map_err(|e: BallistaError| Status::internal(format!("Bad result location: {e}")))?; + let executor = &location.executor_meta; + let failed = |e: BallistaError| { + Status::internal(format!( + "Cannot read a result from {}:{}: {e}", + executor.host, executor.port + )) + }; + let mut client = BallistaClient::try_new( + &executor.host, + executor.port, + config.grpc_client_max_message_size(), + false, + None, + config.io_retries_times() as u8, + config.io_retry_wait_time_ms() as u64, + 0, + 0, + ) + .await + .map_err(failed)?; + let stream = client + .fetch_pipelined_partition(&location, config.pipelined_registration_timeout()) + .await + .map_err(failed)?; + datafusion::physical_plan::common::collect(stream) + .await + .map_err(|e| failed(BallistaError::DataFusionError(Box::new(e)))) +} + +/// The results of a pipelined query, which the scheduler serves itself. +struct StoredResult { + schema: SchemaRef, + partitions: Vec>, +} + +/// Prefix of a ticket for a result partition the scheduler serves itself. The rest is the job id, +/// a NUL, and the partition's index. +const STORED_RESULT_MAGIC: &str = "COMET_RESULT\0"; + /// The Flight endpoint for a result partition: the executor's location, a ticket the executor's /// Flight service reads as a Ballista partition fetch, and when the partition is deleted. fn endpoint( @@ -264,11 +448,18 @@ impl FlightService for CometFlightService { // A range exchange is sampled by a job of its own while the query is planned, which // blocks the planning thread on this runtime. let scheduler = self.scheduler.clone(); + let session = self.session.clone(); + let pipelined = self.pipelined(); let runtime = tokio::runtime::Handle::current(); let plan = tokio::task::spawn_blocking(move || { let mut sampler = |plan| { + // A pipelined job can't range-partition, so an empty sample, which yields no + // split points, has the range exchange read as one partition. + if pipelined { + return Ok(vec![]); + } runtime - .block_on(collect(&scheduler, plan)) + .block_on(collect(&scheduler, &session, plan)) .map_err(|e| DataFusionError::External(Box::new(e))) }; build_plan(query, &mut sampler) @@ -276,11 +467,27 @@ impl FlightService for CometFlightService { .await .map_err(|e| Status::internal(e.to_string()))? .map_err(planning_status)?; - let job = run(&self.scheduler, plan).await?; - let expires_at = self.results_expire_at(job.ended_at); let mut info = FlightInfo::new(); - for location in &job.partition_location { - info = info.with_endpoint(endpoint(location, expires_at)?); + if pipelined { + let (job_id, result) = run_pipelined(&self.scheduler, &self.session, plan).await?; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |now| now.as_millis() as u64); + let expires_at = self.results_expire_at(now); + // No location: the driver reads them from this service. + for index in 0..result.partitions.len() { + let ticket = format!("{STORED_RESULT_MAGIC}{job_id}\0{index}"); + let mut endpoint = FlightEndpoint::new().with_ticket(Ticket::new(ticket)); + endpoint.expiration_time = expires_at; + info = info.with_endpoint(endpoint); + } + self.store(job_id, result); + } else { + let job = run(&self.scheduler, &self.session, plan).await?; + let expires_at = self.results_expire_at(job.ended_at); + for location in &job.partition_location { + info = info.with_endpoint(endpoint(location, expires_at)?); + } } Ok(Response::new(info)) } @@ -313,11 +520,38 @@ impl FlightService for CometFlightService { unimplemented("GetSchema") } + /// Serves a result partition of a pipelined query. A blocking query's results are read from + /// the executors instead. async fn do_get( &self, - _request: Request, + request: Request, ) -> Result, Status> { - unimplemented("DoGet; fetch result partitions from the executors") + let ticket = request.into_inner().ticket; + let ticket = std::str::from_utf8(&ticket) + .ok() + .and_then(|ticket| ticket.strip_prefix(STORED_RESULT_MAGIC)) + .and_then(|ticket| ticket.split_once('\0')) + .and_then(|(job_id, index)| Some((job_id.to_string(), index.parse::().ok()?))); + let Some((job_id, index)) = ticket else { + return unimplemented("DoGet for this ticket; fetch it from an executor"); + }; + let result = self + .stored_results + .lock() + .unwrap() + .get(&job_id) + .cloned() + .ok_or_else(|| { + Status::not_found(format!("The results of job {job_id} have expired")) + })?; + let batches = result.partitions.get(index).cloned().ok_or_else(|| { + Status::not_found(format!("Job {job_id} has no result partition {index}")) + })?; + let stream = FlightDataEncoderBuilder::new() + .with_schema(Arc::clone(&result.schema)) + .build(futures::stream::iter(batches.into_iter().map(Ok))) + .map_err(Status::from); + Ok(Response::new(Box::pin(stream))) } async fn do_put( From 39e08220ef52ad5c7e35165bbe355a5a66b69959 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 31 Oct 2026 13:00:59 -0600 Subject: [PATCH 18/36] style: format the driver's Flight client --- native/core/src/execution/native_executors/client.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/native/core/src/execution/native_executors/client.rs b/native/core/src/execution/native_executors/client.rs index d5360f54fe4..0eea7fb60ae 100644 --- a/native/core/src/execution/native_executors/client.rs +++ b/native/core/src/execution/native_executors/client.rs @@ -84,10 +84,10 @@ async fn submit(scheduler_url: &str, mut query: Vec) -> CometResult Date: Sat, 31 Oct 2026 13:13:01 -0600 Subject: [PATCH 19/36] fix: compile the TPC-H tests of CometNativeExecutorsSuite on Spark 3.x withSQLConf returns Unit on Spark 3.x, so Spark's expected rows come out through a var. --- .../org/apache/comet/exec/CometNativeExecutorsSuite.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala index 6e321279aa6..694702d475b 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala @@ -321,11 +321,13 @@ class CometNativeExecutorsSuite extends CometTestBase with AdaptiveSparkPlanHelp // Spark's own answer. Comet on the driver would run the whole query in one JVM, in the // test's small native memory pool, and Spark's broadcast relations don't fit in the test // JVM's heap either. - val expected = withSQLConf( + // withSQLConf returns Unit on Spark 3.x, so the rows come out through a var. + var expected: Seq[Row] = Nil + withSQLConf( CometConf.COMET_ENABLED.key -> "false", SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") { - sql(query).collect().toSeq + expected = sql(query).collect().toSeq } checkClusterReturns(sql(query), expected, query.toLowerCase.contains("order by")) } finally { From fdb4db09b6be672ad382ccc1f2ead9524933c59c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 31 Oct 2026 14:38:47 -0600 Subject: [PATCH 20/36] feat: let comet-scheduler advertise an external host to its executors Executors report task status to the host the scheduler advertises, which is localhost by default. On Kubernetes they need the scheduler's Service name, which COMET_SCHEDULER_EXTERNAL_HOST sets. --- native/native-executors/src/bin/comet-scheduler.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/native/native-executors/src/bin/comet-scheduler.rs b/native/native-executors/src/bin/comet-scheduler.rs index 1b35588a5c3..4a34a0a1b6d 100644 --- a/native/native-executors/src/bin/comet-scheduler.rs +++ b/native/native-executors/src/bin/comet-scheduler.rs @@ -20,6 +20,8 @@ //! //! Configured through environment variables, with Ballista's defaults otherwise: //! - `COMET_SCHEDULER_BIND_HOST` (default `127.0.0.1`) +//! - `COMET_SCHEDULER_EXTERNAL_HOST`: the host executors reach this scheduler at, which they +//! report task status to (default `localhost`) //! - `COMET_SCHEDULER_PORT`: Ballista's gRPC port, for executors (default `50050`) //! - `COMET_SCHEDULER_FLIGHT_PORT`: the Flight port, for drivers (default `50060`) //! - `COMET_SCHEDULER_RESULT_RETENTION_SECONDS`: how long a query's results stay readable after @@ -50,6 +52,9 @@ async fn main() -> Result<(), Box> { if let Some(host) = env_var("COMET_SCHEDULER_BIND_HOST")? { config.bind_host = host; } + if let Some(host) = env_var("COMET_SCHEDULER_EXTERNAL_HOST")? { + config.external_host = host; + } if let Some(port) = env_var("COMET_SCHEDULER_PORT")? { config.bind_port = port; } From 23277200eaac95174aca6e428758a30efb408a5d Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 31 Oct 2026 15:06:06 -0600 Subject: [PATCH 21/36] feat: let the Comet Native Executor binaries raise their gRPC message limits A task of a stage that reads files carries each of its partitions' copy of the block, which at TPC-H SF1000 can exceed Ballista's 16 MiB gRPC limit, and the scheduler then loses the executor it can't send the task to. COMET_GRPC_MAX_MESSAGE_SIZE sets the limit for both binaries. --- native/native-executors/src/bin/comet-executor.rs | 6 ++++++ native/native-executors/src/bin/comet-scheduler.rs | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/native/native-executors/src/bin/comet-executor.rs b/native/native-executors/src/bin/comet-executor.rs index f1b7cec6a99..2e23ef8dfa1 100644 --- a/native/native-executors/src/bin/comet-executor.rs +++ b/native/native-executors/src/bin/comet-executor.rs @@ -26,6 +26,8 @@ //! - `COMET_EXECUTOR_GRPC_PORT` (default `50052`) //! - `COMET_SCHEDULER_HOST` (default `localhost`) and `COMET_SCHEDULER_PORT` (default `50050`) //! - `COMET_EXECUTOR_TASKS`: concurrent tasks (default: the number of CPUs) +//! - `COMET_GRPC_MAX_MESSAGE_SIZE`: the largest gRPC message, in bytes, the executor accepts or +//! sends, such as a task with many partitions of a scan (default: Ballista's 16 MiB) //! - `COMET_EXECUTOR_WORK_DIR`: where shuffle and spill files go (default: a temporary directory) use std::sync::Arc; @@ -61,6 +63,10 @@ async fn main() -> Result<(), Box> { config.vcores = tasks; } config.work_dir = env_var("COMET_EXECUTOR_WORK_DIR")?; + if let Some(size) = env_var("COMET_GRPC_MAX_MESSAGE_SIZE")? { + config.grpc_max_decoding_message_size = size; + config.grpc_max_encoding_message_size = size; + } start_executor_process(Arc::new(config)).await?; Ok(()) diff --git a/native/native-executors/src/bin/comet-scheduler.rs b/native/native-executors/src/bin/comet-scheduler.rs index 4a34a0a1b6d..01b51b96c74 100644 --- a/native/native-executors/src/bin/comet-scheduler.rs +++ b/native/native-executors/src/bin/comet-scheduler.rs @@ -26,6 +26,8 @@ //! - `COMET_SCHEDULER_FLIGHT_PORT`: the Flight port, for drivers (default `50060`) //! - `COMET_SCHEDULER_RESULT_RETENTION_SECONDS`: how long a query's results stay readable after //! the query ends, whether or not a driver has read them (default `300`). `0` keeps them. +//! - `COMET_GRPC_MAX_MESSAGE_SIZE`: the largest gRPC message, in bytes, the scheduler sends or +//! accepts, such as a task with many partitions of a scan (default: Ballista's 16 MiB) //! - `COMET_SCHEDULER_BALLISTA_CONFIG`: Ballista settings for every query, as comma-separated //! `key=value` pairs, such as `ballista.execution.mode=pipelined` @@ -55,6 +57,11 @@ async fn main() -> Result<(), Box> { if let Some(host) = env_var("COMET_SCHEDULER_EXTERNAL_HOST")? { config.external_host = host; } + if let Some(size) = env_var("COMET_GRPC_MAX_MESSAGE_SIZE")? { + config.grpc_server_max_decoding_message_size = size; + config.grpc_server_max_encoding_message_size = size; + config.grpc_client_max_message_size = size; + } if let Some(port) = env_var("COMET_SCHEDULER_PORT")? { config.bind_port = port; } From 382ae76f9301d25b5482ad5703d59a0b3923e96c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 31 Oct 2026 15:47:12 -0600 Subject: [PATCH 22/36] fix: leave a command's result plan in Spark under Comet Native Executors A command's plan, once it has run, is a CommandResultExec, which explaining it, as tpcbench does for q15's CREATE VIEW, plans again. The cluster has nothing to run for it, so Comet now leaves it in Spark, as it does the commands themselves. --- .../spark/sql/comet/CometNativeExecutorsExec.scala | 5 +++-- .../comet/exec/CometNativeExecutorsSuite.scala | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala index 07b70a956c8..0874fb60cfc 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala @@ -30,7 +30,7 @@ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, Literal} import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, RangePartitioning, SinglePartition} import org.apache.spark.sql.comet.execution.shuffle.{CometNativeShuffle, CometShuffleExchangeExec} -import org.apache.spark.sql.execution.{ColumnarToRowExec, DeserializeToObjectExec, LeafExecNode, ScalarSubquery, SparkPlan} +import org.apache.spark.sql.execution.{ColumnarToRowExec, CommandResultExec, DeserializeToObjectExec, LeafExecNode, ScalarSubquery, SparkPlan} import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, QueryStageExec} import org.apache.spark.sql.execution.command.{DataWritingCommandExec, ExecutedCommandExec} import org.apache.spark.sql.execution.datasources.v2.V2CommandExec @@ -186,7 +186,8 @@ object CometNativeExecutors { !(conf.adaptiveExecutionEnabled && !queryStagePrep && plan.isInstanceOf[Exchange]) && !plan.exists { case _: CometNativeExecutorsExec | _: AdaptiveSparkPlanExec | _: QueryStageExec | - _: ExecutedCommandExec | _: DataWritingCommandExec | _: V2CommandExec => + _: CommandResultExec | _: ExecutedCommandExec | _: DataWritingCommandExec | + _: V2CommandExec => true case _ => false } diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala index 694702d475b..5d53eebd2c4 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala @@ -351,6 +351,20 @@ class CometNativeExecutorsSuite extends CometTestBase with AdaptiveSparkPlanHelp } } + nativeExecutorsTest("a command's plan stays in Spark") { + withTable { path => + onCluster { + withTempView("t") { + spark.read.parquet(path).createOrReplaceTempView("t") + // Explaining a command explains its result, which isn't a query to run on the cluster. + val df = sql("create or replace temp view v as select * from t") + assert( + df.queryExecution.explainString(org.apache.spark.sql.execution.SimpleMode).nonEmpty) + } + } + } + } + nativeExecutorsTest("a query the cluster cannot run fails, naming the operator") { withTable { path => // With Comet's sort off, the sort stays in Spark. From 89251149c0f27ec110afc39f39b46d1add66c844 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 31 Oct 2026 16:19:26 -0600 Subject: [PATCH 23/36] feat: let comet-executor set the size of its memory pool COMET_EXECUTOR_MEMORY_POOL_SIZE sets the bytes the executor's tasks share. Ballista's default, 70% of the container's limit, leaves too little for the memory outside the pool at TPC-H SF1000, where an executor was OOM-killed. --- native/native-executors/src/bin/comet-executor.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/native/native-executors/src/bin/comet-executor.rs b/native/native-executors/src/bin/comet-executor.rs index 2e23ef8dfa1..9a08a7c62df 100644 --- a/native/native-executors/src/bin/comet-executor.rs +++ b/native/native-executors/src/bin/comet-executor.rs @@ -28,6 +28,9 @@ //! - `COMET_EXECUTOR_TASKS`: concurrent tasks (default: the number of CPUs) //! - `COMET_GRPC_MAX_MESSAGE_SIZE`: the largest gRPC message, in bytes, the executor accepts or //! sends, such as a task with many partitions of a scan (default: Ballista's 16 MiB) +//! - `COMET_EXECUTOR_MEMORY_POOL_SIZE`: the bytes of memory the executor's tasks share (default: +//! Ballista's, 70% of the container's memory limit). Memory outside the pool, such as batches +//! in flight between stages, comes on top of it. //! - `COMET_EXECUTOR_WORK_DIR`: where shuffle and spill files go (default: a temporary directory) use std::sync::Arc; @@ -63,6 +66,9 @@ async fn main() -> Result<(), Box> { config.vcores = tasks; } config.work_dir = env_var("COMET_EXECUTOR_WORK_DIR")?; + if let Some(size) = env_var("COMET_EXECUTOR_MEMORY_POOL_SIZE")? { + config.memory_pool_size = Some(size); + } if let Some(size) = env_var("COMET_GRPC_MAX_MESSAGE_SIZE")? { config.grpc_max_decoding_message_size = size; config.grpc_max_encoding_message_size = size; From 8b5f114a42f46925812d4fa3cf02385caea3b119 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 31 Oct 2026 17:34:38 -0600 Subject: [PATCH 24/36] feat: sort each partition and merge for a global sort on Comet Native Executors Spark plans a global sort as a range exchange followed by a sort of each partition, and its driver samples the exchange's input for the bounds. The scheduler did the same, with a sampling job that ran every stage below the exchange once more, which at TPC-H SF1000 more than doubled q4 and q5. The driver now sends a stage that reads a range exchange with the order its block sorts by. The cluster reads the producer's partitions as they are, sorts each with the stage's block, and merges them with a SortPreservingMergeExec, as DataFusion plans a global sort. That needs no sampling, works in pipelined mode too, and leaves only the merge in one task. A stage whose order isn't over columns of types Arrow and Spark order alike still reads the exchange as one partition. SampleExec, the sampling job and their tests are gone. --- .../execution/native_executors/fragment.rs | 2 +- .../src/execution/native_executors/mod.rs | 2 +- native/native-executors/README.md | 22 +- native/native-executors/src/assembly.rs | 165 +++---- native/native-executors/src/codec.rs | 28 +- native/native-executors/src/frontend.rs | 108 +---- native/native-executors/src/lib.rs | 1 - native/native-executors/src/range.rs | 430 ------------------ native/native-executors/tests/assembly.rs | 92 ++-- native/native-executors/tests/cluster.rs | 37 +- native/native-executors/tests/common/mod.rs | 45 +- native/proto/src/proto/native_executors.proto | 13 +- .../sql/comet/CometNativeExecutorsExec.scala | 19 +- 13 files changed, 200 insertions(+), 764 deletions(-) delete mode 100644 native/native-executors/src/range.rs diff --git a/native/core/src/execution/native_executors/fragment.rs b/native/core/src/execution/native_executors/fragment.rs index 34d955a930d..afa10645955 100644 --- a/native/core/src/execution/native_executors/fragment.rs +++ b/native/core/src/execution/native_executors/fragment.rs @@ -169,7 +169,7 @@ impl CometFragmentExec { /// The cast to those types keeps a column nullable wherever the block's output is, even where /// Spark knows it has no nulls, such as a join key a filter has checked, so a fragment can't /// promise more. Nothing between stages relies on a column having no nulls. -fn fragment_schema(output_schema: &[SparkStructField]) -> SchemaRef { +pub fn fragment_schema(output_schema: &[SparkStructField]) -> SchemaRef { let declared = convert_spark_types_to_arrow_schema(output_schema); let fields: Vec = declared .fields() diff --git a/native/core/src/execution/native_executors/mod.rs b/native/core/src/execution/native_executors/mod.rs index f7c152d3fba..48c1405bad5 100644 --- a/native/core/src/execution/native_executors/mod.rs +++ b/native/core/src/execution/native_executors/mod.rs @@ -28,7 +28,7 @@ mod client; mod fragment; -pub use fragment::{block_schema, partition_descriptor, CometFragmentExec}; +pub use fragment::{block_schema, fragment_schema, partition_descriptor, CometFragmentExec}; /// The version of this Comet build. A Comet Native Executor cluster requires a driver's to match /// its own, together with [`datafusion_comet_proto::PROTO_FINGERPRINT`]. diff --git a/native/native-executors/README.md b/native/native-executors/README.md index 5af9d2b2d1f..25bf73afe9c 100644 --- a/native/native-executors/README.md +++ b/native/native-executors/README.md @@ -48,22 +48,22 @@ A client runs a query through the scheduler's Arrow Flight service: A stage that several other stages read runs once for each of them, since Ballista's planner doesn't share a stage between consumers yet. -A range exchange needs split points, which Spark's driver would find by sampling the exchange's -input. The scheduler does the same while it builds the plan: it runs a job of its own over the -stage that feeds the exchange, which keeps a reservoir sample of the sort keys of each partition, -and cuts the sorted sample into ranges of about the same number of rows. Like Spark's, the sampling -job runs that stage once more. A range exchange whose sort keys aren't all columns is read as one -partition instead, so its sort runs in a single task. +Spark plans a global sort as a range exchange followed by a sort of each partition, and its +driver samples the exchange's input for the ranges' bounds. A cluster doesn't range-partition: +the stage reads the producer's partitions as they are, sorts each, and merges the sorted +partitions in the order the driver sends with the stage, as DataFusion plans a global sort. A +stage whose order isn't over columns of types Arrow orders as Spark does, such as floats, reads +the exchange as one partition and sorts it in a single task. Each stage runs on the executors as a `CometFragmentExec` (`native/core/src/execution/native_executors`), which plans the stage's block with Comet's planner for each partition. Ballista moves the data between stages: -| Exchange | Ballista plan | -| --------------------------- | ----------------------------------------------------------------------------------------------------------- | -| Hash | `RepartitionExec(Hash)`: Ballista's hash shuffle, with DataFusion's hash | -| Single partition, broadcast | `CoalescePartitionsExec` | -| Range | `RepartitionExec(Range)`: Ballista's range shuffle, with split points from a sample of the exchange's input | +| Exchange | Ballista plan | +| --------------------------- | -------------------------------------------------------------------------------------------- | +| Hash | `RepartitionExec(Hash)`: Ballista's hash shuffle, with DataFusion's hash | +| Single partition, broadcast | `CoalescePartitionsExec` | +| Range | None: the stage sorts the producer's partitions, and a `SortPreservingMergeExec` merges them | ## Building and running diff --git a/native/native-executors/src/assembly.rs b/native/native-executors/src/assembly.rs index 541a3fac3a9..95f26d9e2ed 100644 --- a/native/native-executors/src/assembly.rs +++ b/native/native-executors/src/assembly.rs @@ -21,16 +21,16 @@ use std::collections::HashMap; use std::sync::Arc; use comet::execution::native_executors::{ - block_schema, partition_descriptor, CometFragmentExec, COMET_VERSION, + block_schema, fragment_schema, partition_descriptor, CometFragmentExec, COMET_VERSION, }; -use datafusion::arrow::array::RecordBatch; use datafusion::arrow::compute::SortOptions; use datafusion::arrow::datatypes::{DataType, SchemaRef}; use datafusion::common::{plan_datafusion_err, plan_err, Result}; use datafusion::physical_expr::expressions::Column; -use datafusion::physical_expr::{LexOrdering, PhysicalExpr, PhysicalSortExpr, RangePartitioning}; +use datafusion::physical_expr::{LexOrdering, PhysicalExpr, PhysicalSortExpr}; use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; use datafusion::physical_plan::repartition::RepartitionExec; +use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; use datafusion::physical_plan::ExecutionPlanProperties; use datafusion::physical_plan::{ExecutionPlan, Partitioning}; use datafusion_comet_proto::spark_expression::expr::ExprStruct; @@ -38,15 +38,9 @@ use datafusion_comet_proto::spark_expression::{Expr, NullOrdering, SortDirection use datafusion_comet_proto::spark_native_executors::{Fragment, QueryRequest, Stage, StageInput}; use datafusion_comet_proto::spark_partitioning::partitioning::PartitioningStruct; use datafusion_comet_proto::PROTO_FINGERPRINT; -use log::{info, warn}; +use log::warn; use prost::Message; -use crate::range::{sample_size_per_partition, split_points, SampleExec}; - -/// Runs a plan to completion and returns the rows of all its partitions: how the scheduler -/// samples the input of a range exchange. -pub type Sampler<'a> = dyn FnMut(Arc) -> Result> + 'a; - /// Checks that `request` comes from a driver running this Comet build. The plan format is not /// yet stable across versions, and an executor would ignore any field it doesn't know. pub fn check_version(request: &QueryRequest) -> Result<()> { @@ -63,17 +57,15 @@ pub fn check_version(request: &QueryRequest) -> Result<()> { } /// Builds the physical plan of `request`: a [`CometFragmentExec`] per stage, with a node that -/// Ballista's planner turns into a stage boundary on each edge. The root is the last stage. Each -/// range exchange is range-partitioned by split points found by sampling its input with -/// `sampler`, as Spark does. +/// Ballista's planner turns into a stage boundary on each edge. The root is the last stage. /// /// Each stage's block is planned with Comet's planner to check it before anything runs. That can /// block, so don't call this on an async runtime's thread. A stage that several stages read runs /// once for each of them, since Ballista's planner doesn't share a stage between consumers yet. -pub fn build_plan(request: QueryRequest, sampler: &mut Sampler) -> Result> { +pub fn build_plan(request: QueryRequest) -> Result> { let mut stages: Vec> = Vec::with_capacity(request.stages.len()); for (index, stage) in request.stages.into_iter().enumerate() { - let fragment = build_stage(stage, &stages, &request.conf, sampler) + let fragment = build_stage(stage, &stages, &request.conf) .map_err(|e| e.context(format!("Stage {index} of the query")))?; stages.push(fragment); } @@ -86,12 +78,17 @@ fn build_stage( stage: Stage, producers: &[Arc], conf: &HashMap, - sampler: &mut Sampler, ) -> Result> { let plan = stage .plan .as_ref() .ok_or_else(|| plan_datafusion_err!("The stage has no plan"))?; + // A stage that sorts what a range exchange delivers sorts the producer's partitions as they + // are, and the sorted partitions are merged, as DataFusion plans a global sort. + let merge = merge_ordering( + &stage.output_ordering, + &fragment_schema(&stage.output_schema), + ); let mut inputs = Vec::with_capacity(stage.inputs.len()); for input in &stage.inputs { let producer = producers.get(input.producer as usize).ok_or_else(|| { @@ -100,7 +97,7 @@ fn build_stage( input.producer ) })?; - inputs.push(exchange(input, Arc::clone(producer), sampler)?); + inputs.push(exchange(input, Arc::clone(producer), merge.is_some())?); } // A stage that reads exchanges has as many partitions as they deliver, and they must agree. let mut partitioned = stage @@ -152,10 +149,14 @@ fn build_stage( output_schema: stage.output_schema, conf: conf.clone(), }; - Ok(Arc::new(CometFragmentExec::try_new( - Arc::new(fragment), - children, - )?)) + let fragment: Arc = + Arc::new(CometFragmentExec::try_new(Arc::new(fragment), children)?); + Ok(match merge { + Some(ordering) if num_partitions > 1 => { + Arc::new(SortPreservingMergeExec::new(ordering, fragment)) + } + _ => fragment, + }) } /// The exchange that carries `producer`'s output to a stage, as a node Ballista's planner turns @@ -165,10 +166,14 @@ fn build_stage( /// consistent within a query, which is all joins, aggregates and windows need, but a plan that /// relies on Spark's own placement of rows, such as a join with a bucketed table, would be wrong /// (https://github.com/apache/datafusion-comet/issues/7368). +/// +/// A range exchange whose stage merges its sorted partitions is no exchange at all: the stage +/// reads the producer's partitions as they are. Otherwise it is read as one partition, and the +/// stage sorts it in a single task. fn exchange( input: &StageInput, producer: Arc, - sampler: &mut Sampler, + merged: bool, ) -> Result> { if input.broadcast { return Ok(one_partition(producer)); @@ -197,13 +202,8 @@ fn exchange( Partitioning::Hash(keys, hash.num_partitions as usize), )?)) } - Some(PartitioningStruct::RangePartition(range)) => { - Ok(match range_exchange(range, &producer, sampler)? { - Some(partitioning) => Arc::new(RepartitionExec::try_new(producer, partitioning)?), - // Read as one partition, the stage sorts correctly in a single task. - None => one_partition(producer), - }) - } + Some(PartitioningStruct::RangePartition(_)) if merged => Ok(producer), + Some(PartitioningStruct::RangePartition(_)) => Ok(one_partition(producer)), Some(PartitioningStruct::SinglePartition(_)) => Ok(one_partition(producer)), other => plan_err!("Comet Native Executors don't support this exchange yet: {other:?}"), } @@ -228,11 +228,10 @@ fn bound_column(expr: &Expr, num_fields: usize) -> Option { } } -/// Whether Arrow orders values of `data_type` as Spark does, so that a range shuffle, which -/// compares raw values, puts rows in the ranges that Comet's sort orders them by. Comet's sort -/// normalizes floats first (a NaN of either sign sorts last), so a float key, or a nested one -/// that may hold floats, can't be range-partitioned this way -/// (https://github.com/apache/datafusion-comet/issues/7366). +/// Whether Arrow orders values of `data_type` as Spark does, so that a merge, which compares raw +/// values, keeps the order Comet's sort put each partition in. Comet's sort normalizes floats +/// first (a NaN of either sign sorts last), so a float key, or a nested one that may hold +/// floats, can't be merged this way (https://github.com/apache/datafusion-comet/issues/7366). fn orders_like_spark(data_type: &DataType) -> bool { data_type.is_integer() || matches!( @@ -254,74 +253,32 @@ fn orders_like_spark(data_type: &DataType) -> bool { ) } -/// The range partitioning of a range exchange over `producer`, with split points found by -/// sampling it. `None` when the exchange should be read as one partition instead: when a sort -/// key is not a column, or of a type a range shuffle doesn't order as Spark does, or the sample -/// yields no split points. -/// -/// The sampling job runs the stages below the exchange once more -/// (https://github.com/apache/datafusion-comet/issues/7367). -fn range_exchange( - range: &datafusion_comet_proto::spark_partitioning::RangePartition, - producer: &Arc, - sampler: &mut Sampler, -) -> Result> { - let partitions = range.num_partitions.max(1) as usize; - if partitions == 1 { - return Ok(None); - } - let schema = producer.schema(); - let mut keys = Vec::with_capacity(range.sort_orders.len()); - let mut sort_exprs = Vec::with_capacity(range.sort_orders.len()); - for order in &range.sort_orders { - let key = match order.expr_struct.as_ref() { - Some(ExprStruct::SortOrder(order)) => order.child.as_deref().and_then(|child| { - bound_column(child, schema.fields().len()).map(|index| (index, order)) - }), - _ => None, - }; - let Some((index, order)) = key else { - warn!("A range exchange sorts by an expression that is not a column, so it runs in one task"); - return Ok(None); - }; - let data_type = schema.field(index).data_type(); - if !orders_like_spark(data_type) { - warn!("A range exchange sorts by a {data_type} column, so it runs in one task"); - return Ok(None); - } - keys.push(index); - sort_exprs.push(PhysicalSortExpr::new( - column(&schema, index), - SortOptions { - descending: order.direction == SortDirection::Descending as i32, - nulls_first: order.null_ordering == NullOrdering::NullsFirst as i32, - }, - )); - } - let options: Vec = sort_exprs.iter().map(|expr| expr.options).collect(); - let Some(ordering) = LexOrdering::new(sort_exprs) else { - return Ok(None); - }; - - let per_partition = - sample_size_per_partition(partitions, producer.output_partitioning().partition_count()); - let sample = Arc::new(SampleExec::try_new( - Arc::clone(producer), - keys, - per_partition, - )?); - let sample_schema = sample.schema(); - let samples = sampler(sample)?; - let points = split_points(&sample_schema, &samples, &options, partitions)?; - if points.is_empty() { - return Ok(None); - } - info!( - "Range-partitioning an exchange into {} partitions by a sample of {} rows", - points.len() + 1, - samples.iter().map(RecordBatch::num_rows).sum::() - ); - Ok(Some(Partitioning::Range(RangePartitioning::try_new( - ordering, points, - )?))) +/// The order to merge a stage's sorted partitions in: its declared output ordering, when every +/// key is a column of a type whose order Arrow and Spark agree on. `None` when there's nothing +/// to merge by, and the stage then reads its range exchange as one partition. +fn merge_ordering(orders: &[Expr], schema: &SchemaRef) -> Option { + let sort_exprs = orders + .iter() + .map(|order| { + let Some(ExprStruct::SortOrder(order)) = order.expr_struct.as_ref() else { + return None; + }; + let index = bound_column(order.child.as_deref()?, schema.fields().len())?; + if !orders_like_spark(schema.field(index).data_type()) { + warn!( + "A global sort by a {} column runs in one task", + schema.field(index).data_type() + ); + return None; + } + Some(PhysicalSortExpr::new( + column(schema, index), + SortOptions { + descending: order.direction == SortDirection::Descending as i32, + nulls_first: order.null_ordering == NullOrdering::NullsFirst as i32, + }, + )) + }) + .collect::>>()?; + LexOrdering::new(sort_exprs) } diff --git a/native/native-executors/src/codec.rs b/native/native-executors/src/codec.rs index f0de2b712f6..b8b9a7f9320 100644 --- a/native/native-executors/src/codec.rs +++ b/native/native-executors/src/codec.rs @@ -24,19 +24,14 @@ use comet::execution::native_executors::CometFragmentExec; use datafusion::common::{internal_datafusion_err, Result}; use datafusion::execution::TaskContext; use datafusion::physical_plan::ExecutionPlan; -use datafusion_comet_proto::spark_native_executors::{Fragment, Sample}; +use datafusion_comet_proto::spark_native_executors::Fragment; use datafusion_proto::physical_plan::{PhysicalExtensionCodec, PhysicalProtoConverterExtension}; use prost::Message; -use crate::range::SampleExec; - /// Prefix of an encoded [`CometFragmentExec`], which tells it apart from Ballista's own nodes. /// The rest is its `Fragment`; its children travel separately, as the plan's. const FRAGMENT_MAGIC: &[u8] = b"COMET_FRAGMENT\0"; -/// Prefix of an encoded [`SampleExec`]. The rest is its `Sample`; its input travels separately. -const SAMPLE_MAGIC: &[u8] = b"COMET_SAMPLE\0"; - /// Ballista's physical extension codec, extended with Comet's operators. #[derive(Debug, Default)] pub struct CometPhysicalCodec { @@ -51,18 +46,6 @@ impl PhysicalExtensionCodec for CometPhysicalCodec { ctx: &TaskContext, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result> { - if let Some(buf) = buf.strip_prefix(SAMPLE_MAGIC) { - let sample = Sample::decode(buf) - .map_err(|e| internal_datafusion_err!("Cannot decode a SampleExec: {e}"))?; - let [input] = inputs else { - return Err(internal_datafusion_err!("A SampleExec takes one input")); - }; - return Ok(Arc::new(SampleExec::try_new( - Arc::clone(input), - sample.keys.iter().map(|key| *key as usize).collect(), - sample.per_partition as usize, - )?)); - } let Some(buf) = buf.strip_prefix(FRAGMENT_MAGIC) else { return self.ballista.try_decode(buf, inputs, ctx, proto_converter); }; @@ -80,15 +63,6 @@ impl PhysicalExtensionCodec for CometPhysicalCodec { buf: &mut Vec, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result<()> { - if let Some(sample) = node.downcast_ref::() { - buf.extend_from_slice(SAMPLE_MAGIC); - return Sample { - per_partition: sample.per_partition() as u64, - keys: sample.keys().iter().map(|key| *key as u64).collect(), - } - .encode(buf) - .map_err(|e| internal_datafusion_err!("Cannot encode a SampleExec: {e}")); - } let Some(fragment) = node.downcast_ref::() else { return self.ballista.try_encode(node, buf, proto_converter); }; diff --git a/native/native-executors/src/frontend.rs b/native/native-executors/src/frontend.rs index a2a7940789d..f457cd04d27 100644 --- a/native/native-executors/src/frontend.rs +++ b/native/native-executors/src/frontend.rs @@ -44,14 +44,13 @@ use ballista_core::serde::scheduler::{Action as BallistaAction, ShuffleFileKind} use ballista_scheduler::scheduler_server::SchedulerServer; use datafusion::arrow::array::RecordBatch; use datafusion::arrow::datatypes::SchemaRef; -use datafusion::common::DataFusionError; use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties}; use datafusion::prelude::SessionConfig; use datafusion_comet_proto::spark_native_executors::QueryRequest; use datafusion_proto::protobuf::{LogicalPlanNode, PhysicalPlanNode}; use futures::stream::BoxStream; -use futures::{StreamExt, TryStreamExt}; +use futures::TryStreamExt; use log::info; use prost::Message; use prost_types::Timestamp; @@ -97,7 +96,7 @@ impl CometFlightService { } /// Whether queries run in Ballista's pipelined mode, which streams batches between stages - /// instead of writing shuffle files, and can't range-partition. + /// instead of writing shuffle files. fn pipelined(&self) -> bool { matches!( self.session.ballista_config().execution_mode(), @@ -145,69 +144,6 @@ impl CometFlightService { } } -/// Runs `plan` as a Ballista job on `scheduler` and reads the rows of all its partitions back from -/// the executors that hold them, a few partitions at a time. -async fn collect( - scheduler: &SchedulerServer, - session: &SessionConfig, - plan: Arc, -) -> Result, Status> { - let job = run(scheduler, session, plan).await?; - let config = BallistaConfig::default(); - let batches: Vec> = futures::stream::iter(job.partition_location) - .map(|location| fetch(location, &config)) - .buffer_unordered(FETCH_CONCURRENCY) - .try_collect() - .await?; - Ok(batches.concat()) -} - -/// How many sample partitions the scheduler reads at once. -const FETCH_CONCURRENCY: usize = 16; - -/// Reads the rows of one partition from the executor that holds it, with Ballista's client. -async fn fetch( - location: PartitionLocation, - config: &BallistaConfig, -) -> Result, Status> { - let layout = location.layout(); - let (Some(executor), Some(partition)) = (location.executor_meta, location.partition_id) else { - return Err(Status::internal("A sample partition has no executor or ID")); - }; - let failed = |e: BallistaError| { - Status::internal(format!( - "Cannot read a sample from {}:{}: {e}", - executor.host, executor.port - )) - }; - let mut client = BallistaClient::try_new( - &executor.host, - executor.port as u16, - config.grpc_client_max_message_size(), - false, - None, - config.io_retries_times() as u8, - config.io_retry_wait_time_ms() as u64, - 0, - 0, - ) - .await - .map_err(failed)?; - let stream = client - .fetch_partition( - &executor.id, - &partition.into(), - location.file_id, - layout, - true, - ) - .await - .map_err(failed)?; - datafusion::physical_plan::common::collect(stream) - .await - .map_err(|e| failed(BallistaError::DataFusionError(Box::new(e)))) -} - /// Runs `plan` as a Ballista job on `scheduler` to completion and returns its result: where its /// output partitions are, in partition order, and when it ended. async fn run( @@ -268,8 +204,8 @@ async fn submit( /// since Spark can compute a partition of an RDD again, so the scheduler reads the streams itself /// and serves them from its own Flight service for as long as a blocking job's results are kept. /// -/// The result is gathered into one partition first. A pipelined query's order lives in one -/// partition anyway, since a pipelined job reads a range exchange as one partition. +/// The result is gathered into one partition first. A sorted query's result is already one +/// partition, the merge of its sorted partitions. async fn run_pipelined( scheduler: &SchedulerServer, session: &SessionConfig, @@ -410,17 +346,6 @@ fn endpoint( Ok(endpoint) } -/// The status for a query the scheduler couldn't plan: a sampling job's own status when that job -/// failed, and otherwise an invalid query. -fn planning_status(error: DataFusionError) -> Status { - if let DataFusionError::External(cause) = error.find_root() { - if let Some(status) = cause.downcast_ref::() { - return status.clone(); - } - } - Status::invalid_argument(error.to_string()) -} - fn unimplemented(method: &str) -> Result { Err(Status::unimplemented(format!( "A Comet scheduler doesn't implement {method}" @@ -445,28 +370,11 @@ impl FlightService for CometFlightService { let query = QueryRequest::decode(descriptor.cmd.as_ref()) .map_err(|e| Status::invalid_argument(format!("Cannot decode the query: {e}")))?; check_version(&query).map_err(|e| Status::failed_precondition(e.to_string()))?; - // A range exchange is sampled by a job of its own while the query is planned, which - // blocks the planning thread on this runtime. - let scheduler = self.scheduler.clone(); - let session = self.session.clone(); let pipelined = self.pipelined(); - let runtime = tokio::runtime::Handle::current(); - let plan = tokio::task::spawn_blocking(move || { - let mut sampler = |plan| { - // A pipelined job can't range-partition, so an empty sample, which yields no - // split points, has the range exchange read as one partition. - if pipelined { - return Ok(vec![]); - } - runtime - .block_on(collect(&scheduler, &session, plan)) - .map_err(|e| DataFusionError::External(Box::new(e))) - }; - build_plan(query, &mut sampler) - }) - .await - .map_err(|e| Status::internal(e.to_string()))? - .map_err(planning_status)?; + let plan = tokio::task::spawn_blocking(move || build_plan(query)) + .await + .map_err(|e| Status::internal(e.to_string()))? + .map_err(|e| Status::invalid_argument(e.to_string()))?; let mut info = FlightInfo::new(); if pipelined { let (job_id, result) = run_pipelined(&self.scheduler, &self.session, plan).await?; diff --git a/native/native-executors/src/lib.rs b/native/native-executors/src/lib.rs index ff0543d2b10..02251c83cb2 100644 --- a/native/native-executors/src/lib.rs +++ b/native/native-executors/src/lib.rs @@ -31,7 +31,6 @@ use std::str::FromStr; pub mod assembly; pub mod codec; pub mod frontend; -pub mod range; /// The value of the environment variable `key`, if it is set, for the binaries' settings. pub fn env_var(key: &str) -> Result, String> diff --git a/native/native-executors/src/range.rs b/native/native-executors/src/range.rs deleted file mode 100644 index 668194907c3..00000000000 --- a/native/native-executors/src/range.rs +++ /dev/null @@ -1,430 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Range exchanges: the split points of a range shuffle, found by sampling the exchange's input -//! as Spark's `RangePartitioner` does. -//! -//! The scheduler runs a sampling job over the stage that feeds a range exchange, a -//! [`SampleExec`] over each of its partitions, and reads every partition's sample back. It then -//! sorts the sample by the exchange's sort keys and takes evenly spaced rows as the split points -//! of a [`RangePartitioning`](datafusion::physical_expr::Partitioning::Range), which Ballista -//! plans as a range shuffle. Like Spark's, the sampling job runs the producing stage once more. - -use std::fmt; -use std::sync::Arc; - -use datafusion::arrow::array::RecordBatch; -use datafusion::arrow::compute::{concat_batches, interleave_record_batch, SortOptions}; -use datafusion::arrow::datatypes::SchemaRef; -use datafusion::arrow::row::{RowConverter, SortField}; -use datafusion::common::tree_node::TreeNodeRecursion; -use datafusion::common::{internal_datafusion_err, Result, ScalarValue, SplitPoint}; -use datafusion::execution::TaskContext; -use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr}; -use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; -use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning, - PlanProperties, SendableRecordBatchStream, -}; -use futures::StreamExt; - -/// How many rows to sample from each of `input_partitions` partitions to range-partition their -/// rows into `output_partitions`, as Spark's `RangePartitioner` decides. -pub fn sample_size_per_partition(output_partitions: usize, input_partitions: usize) -> usize { - let sample_size = (20 * output_partitions).min(1_000_000); - (3 * sample_size).div_ceil(input_partitions.max(1)) -} - -/// A uniform sample of up to `per_partition` rows of each partition of its input, keeping only -/// the columns `keys`, by reservoir sampling. -#[derive(Debug)] -pub struct SampleExec { - input: Arc, - keys: Vec, - per_partition: usize, - schema: SchemaRef, - props: Arc, -} - -impl SampleExec { - pub fn try_new( - input: Arc, - keys: Vec, - per_partition: usize, - ) -> Result { - let schema = Arc::new(input.schema().project(&keys)?); - let props = Arc::new(PlanProperties::new( - EquivalenceProperties::new(Arc::clone(&schema)), - Partitioning::UnknownPartitioning(input.output_partitioning().partition_count()), - EmissionType::Final, - Boundedness::Bounded, - )); - Ok(Self { - input, - keys, - per_partition, - schema, - props, - }) - } - - pub fn keys(&self) -> &[usize] { - &self.keys - } - - pub fn per_partition(&self) -> usize { - self.per_partition - } -} - -impl DisplayAs for SampleExec { - fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result { - write!( - f, - "SampleExec: keys={:?}, per_partition={}", - self.keys, self.per_partition - ) - } -} - -impl ExecutionPlan for SampleExec { - fn name(&self) -> &str { - "SampleExec" - } - - fn properties(&self) -> &Arc { - &self.props - } - - fn children(&self) -> Vec<&Arc> { - vec![&self.input] - } - - /// The keys are column indexes, so there are no expressions to visit. - fn apply_expressions( - &self, - _f: &mut dyn FnMut(&Arc) -> Result, - ) -> Result { - Ok(TreeNodeRecursion::Continue) - } - - fn with_new_children( - self: Arc, - children: Vec>, - ) -> Result> { - let [input] = <[_; 1]>::try_from(children) - .map_err(|_| internal_datafusion_err!("SampleExec takes one input"))?; - Ok(Arc::new(Self::try_new( - input, - self.keys.clone(), - self.per_partition, - )?)) - } - - fn execute( - &self, - partition: usize, - context: Arc, - ) -> Result { - let mut input = self.input.execute(partition, context)?; - let keys = self.keys.clone(); - let capacity = self.per_partition; - let schema = Arc::clone(&self.schema); - let stream = futures::stream::once(async move { - let mut reservoir = RecordBatch::new_empty(Arc::clone(&schema)); - let mut rng = XorShift::new(partition as u64); - // Rows of the partition seen so far. - let mut seen: u64 = 0; - while let Some(batch) = input.next().await { - let batch = batch?.project(&keys)?; - // Where each row of the new reservoir comes from: (0, row) of the reservoir or - // (1, row) of the batch. Built only once the batch changes the reservoir, which - // most batches of a large partition don't. - let mut indices: Option> = None; - let mut size = reservoir.num_rows(); - for row in 0..batch.num_rows() { - let slot = if size < capacity { - size += 1; - size - 1 - } else { - rng.below(seen + 1) as usize - }; - if slot < capacity { - let indices = indices.get_or_insert_with(|| { - (0..reservoir.num_rows()).map(|row| (0, row)).collect() - }); - if slot == indices.len() { - indices.push((1, row)); - } else { - indices[slot] = (1, row); - } - } - seen += 1; - } - if let Some(indices) = indices { - reservoir = interleave_record_batch(&[&reservoir, &batch], &indices)?; - } - } - Ok(reservoir) - }); - Ok(Box::pin(RecordBatchStreamAdapter::new( - Arc::clone(&self.schema), - stream, - ))) - } -} - -/// A small deterministic random number generator, so that a query samples the same rows each -/// time it runs. -struct XorShift(u64); - -impl XorShift { - fn new(seed: u64) -> Self { - // Any nonzero state works; mix the seed so that neighbouring partitions differ. - Self(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1) - } - - fn next(&mut self) -> u64 { - let mut x = self.0; - x ^= x << 13; - x ^= x >> 7; - x ^= x << 17; - self.0 = x; - x - } - - /// A number in `0..bound`. - fn below(&mut self, bound: u64) -> u64 { - self.next() % bound - } -} - -/// The split points that cut `samples`, rows of the exchange's sort keys, into `partitions` -/// ranges of about the same number of rows, in the order `options` gives each key. Equal -/// candidates collapse into one, so there can be fewer than `partitions - 1`, and none when the -/// sample is empty. -pub fn split_points( - schema: &SchemaRef, - samples: &[RecordBatch], - options: &[SortOptions], - partitions: usize, -) -> Result> { - let batch = concat_batches(schema, samples)?; - let rows = batch.num_rows(); - if rows == 0 || partitions <= 1 { - return Ok(vec![]); - } - let converter = RowConverter::new( - schema - .fields() - .iter() - .zip(options) - .map(|(field, options)| { - SortField::new_with_options(field.data_type().clone(), *options) - }) - .collect(), - )?; - // The row format orders the keys as `options` say, so it both sorts the sample and tells - // equal neighbours apart. - let keys = converter.convert_columns(batch.columns())?; - let mut order: Vec = (0..rows).collect(); - order.sort_unstable_by(|a, b| keys.row(*a).cmp(&keys.row(*b))); - let mut points = vec![]; - let mut last = None; - for i in 1..partitions { - let index = order[i * rows / partitions]; - let key = keys.row(index); - // Split points must be strictly ordered. - if last.is_some_and(|last| key <= keys.row(last)) { - continue; - } - last = Some(index); - let values = batch - .columns() - .iter() - .map(|column| ScalarValue::try_from_array(column, index)) - .collect::>>()?; - points.push(SplitPoint::new(values)); - } - Ok(points) -} - -#[cfg(test)] -mod tests { - use super::*; - use datafusion::arrow::array::{Int64Array, StringArray}; - use datafusion::arrow::datatypes::{DataType, Field, Schema}; - use datafusion::datasource::memory::MemorySourceConfig; - use datafusion::physical_plan::collect; - use datafusion::prelude::SessionContext; - - fn longs(values: Vec>) -> (SchemaRef, RecordBatch) { - let schema = Arc::new(Schema::new(vec![Field::new("k", DataType::Int64, true)])); - let batch = RecordBatch::try_new( - Arc::clone(&schema), - vec![Arc::new(Int64Array::from(values))], - ) - .unwrap(); - (schema, batch) - } - - fn values(points: &[SplitPoint]) -> Vec> { - points.iter().map(|point| point.values().to_vec()).collect() - } - - #[test] - fn split_points_cut_a_sample_into_equal_ranges() { - let (schema, batch) = longs((0..100).rev().map(Some).collect()); - let points = split_points(&schema, &[batch], &[SortOptions::default()], 4).unwrap(); - assert_eq!( - values(&points), - [25, 50, 75].map(|v| vec![ScalarValue::Int64(Some(v))]) - ); - } - - #[test] - fn split_points_follow_the_sort_direction_and_null_order() { - let mut keys: Vec> = (0..6).map(Some).collect(); - keys.extend([None, None]); - let (schema, batch) = longs(keys); - let descending = SortOptions { - descending: true, - nulls_first: true, - }; - let points = split_points(&schema, &[batch], &[descending], 4).unwrap(); - // Sorted: null, null, 5, 4, 3, 2, 1, 0. - assert_eq!( - values(&points), - [Some(5), Some(3), Some(1)].map(|v| vec![ScalarValue::Int64(v)]) - ); - } - - #[test] - fn equal_candidates_collapse_into_one_split_point() { - let (schema, batch) = longs([1, 1, 1, 1, 1, 1, 2, 2].map(Some).to_vec()); - let points = split_points(&schema, &[batch], &[SortOptions::default()], 4).unwrap(); - assert_eq!( - values(&points), - [1, 2].map(|v| vec![ScalarValue::Int64(Some(v))]) - ); - } - - #[test] - fn an_empty_sample_has_no_split_points() { - let (schema, _) = longs(vec![]); - let points = split_points(&schema, &[], &[SortOptions::default()], 4).unwrap(); - assert!(points.is_empty()); - } - - #[test] - fn split_points_order_compound_keys_lexicographically() { - let schema = Arc::new(Schema::new(vec![ - Field::new("a", DataType::Utf8, false), - Field::new("b", DataType::Int64, false), - ])); - let batch = RecordBatch::try_new( - Arc::clone(&schema), - vec![ - Arc::new(StringArray::from(vec!["x", "x", "y", "y"])), - Arc::new(Int64Array::from(vec![2, 1, 2, 1])), - ], - ) - .unwrap(); - let options = [ - SortOptions::default(), - SortOptions { - descending: true, - nulls_first: false, - }, - ]; - let points = split_points(&schema, &[batch], &options, 2).unwrap(); - // Sorted: (x, 2), (x, 1), (y, 2), (y, 1). - assert_eq!( - values(&points), - [vec![ - ScalarValue::Utf8(Some("y".to_string())), - ScalarValue::Int64(Some(2)) - ]] - ); - } - - #[tokio::test] - async fn sample_keeps_up_to_its_size_from_each_partition() { - let schema = Arc::new(Schema::new(vec![ - Field::new("k", DataType::Int64, false), - Field::new("v", DataType::Utf8, false), - ])); - let batch = |start: i64| { - RecordBatch::try_new( - Arc::clone(&schema), - vec![ - Arc::new(Int64Array::from_iter_values(start..start + 50)), - Arc::new(StringArray::from_iter_values( - (start..start + 50).map(|v| v.to_string()), - )), - ], - ) - .unwrap() - }; - // Two partitions: one of 100 rows in two batches, and one of 3 rows. - let partitions = vec![vec![batch(0), batch(50)], vec![batch(1000).slice(0, 3)]]; - let input = - MemorySourceConfig::try_new_exec(&partitions, Arc::clone(&schema), None).unwrap(); - let sample: Arc = - Arc::new(SampleExec::try_new(input, vec![0], 10).unwrap()); - let ctx = SessionContext::new(); - - let mut sizes = vec![]; - for partition in 0..2 { - let batches = datafusion::physical_plan::common::collect( - sample.execute(partition, ctx.task_ctx()).unwrap(), - ) - .await - .unwrap(); - let rows: Vec = batches - .iter() - .flat_map(|batch| { - assert_eq!(batch.num_columns(), 1); - batch - .column(0) - .as_any() - .downcast_ref::() - .unwrap() - .values() - .to_vec() - }) - .collect(); - let mut distinct = rows.clone(); - distinct.sort(); - distinct.dedup(); - assert_eq!(distinct.len(), rows.len(), "{rows:?} repeats a row"); - sizes.push(rows.len()); - } - assert_eq!(sizes, [10, 3]); - // The whole plan collects the same rows. - assert_eq!( - collect(sample, ctx.task_ctx()) - .await - .unwrap() - .iter() - .map(RecordBatch::num_rows) - .sum::(), - 13 - ); - } -} diff --git a/native/native-executors/tests/assembly.rs b/native/native-executors/tests/assembly.rs index 1f27b2a9007..b57f575a80d 100644 --- a/native/native-executors/tests/assembly.rs +++ b/native/native-executors/tests/assembly.rs @@ -22,10 +22,11 @@ mod common; use comet_native_executors::assembly::build_plan; use common::*; +use datafusion::arrow::array::AsArray; use datafusion::arrow::datatypes::{DataType, Int32Type, Int64Type}; use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; -use datafusion::physical_plan::repartition::RepartitionExec; -use datafusion::physical_plan::{collect, ExecutionPlan, ExecutionPlanProperties, Partitioning}; +use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; +use datafusion::physical_plan::{collect, ExecutionPlan, ExecutionPlanProperties}; use datafusion::prelude::SessionContext; use datafusion_comet_proto::spark_expression::{ expr::ExprStruct, literal::Value, Expr, IfExpr, Literal, Subquery, @@ -38,29 +39,10 @@ use datafusion_comet_proto::spark_operator::{ use prost::Message; use std::sync::Arc; -/// Assembles `request`, running each sampling plan in this process. async fn assemble(request: QueryRequest) -> datafusion::common::Result> { - assemble_counting_samples(request) + tokio::task::spawn_blocking(move || build_plan(request)) .await - .map(|(plan, _)| plan) -} - -/// Assembles `request` like [`assemble`], and also returns how many sampling plans ran. -async fn assemble_counting_samples( - request: QueryRequest, -) -> datafusion::common::Result<(Arc, usize)> { - let runtime = tokio::runtime::Handle::current(); - tokio::task::spawn_blocking(move || { - let mut samples = 0; - let mut sampler = |plan: Arc| { - samples += 1; - runtime.block_on(collect(plan, SessionContext::new().task_ctx())) - }; - let plan = build_plan(request, &mut sampler)?; - Ok((plan, samples)) - }) - .await - .unwrap() + .unwrap() } /// Runs `request` and returns its single column, sorted. @@ -97,8 +79,8 @@ async fn single_partition_exchange_collects_every_row() { } #[tokio::test(flavor = "multi_thread")] -async fn range_exchange_without_column_keys_is_read_as_one_partition() { - // The driver asks for the exchange's partition count, but sends no sort key to sample. +async fn range_exchange_without_an_ordering_is_read_as_one_partition() { + // A stage that declares no ordering has nothing to merge its partitions by. let request = query(vec![ range_stage(100, 4), reading_stage(0, range_partitioning(4, vec![]), 4), @@ -108,46 +90,50 @@ async fn range_exchange_without_column_keys_is_read_as_one_partition() { assert_eq!(run(request).await, (0..100).collect::>()); } -/// The exchange a reading stage reads: its second child, after the partition descriptor. -fn exchange_of(plan: &Arc) -> Arc { - Arc::clone(plan.children()[1]) -} - #[tokio::test(flavor = "multi_thread")] -async fn range_exchange_is_range_partitioned_by_a_sample() { +async fn range_exchange_is_sorted_in_each_partition_and_merged() { + let order = first_column_descending(long_type()); let request = query(vec![ range_stage(1000, 4), - reading_stage(0, range_partitioning(4, vec![first_column_descending()]), 4), + sorting_stage(0, range_partitioning(4, vec![]), order, long_type()), ]); - let (plan, samples) = assemble_counting_samples(request).await.unwrap(); - assert_eq!(samples, 1); - assert_eq!(plan.output_partitioning().partition_count(), 4); - let exchange = exchange_of(&plan); - let repartition = exchange - .downcast_ref::() - .expect("a range exchange becomes a RepartitionExec"); - let Partitioning::Range(range) = repartition.partitioning() else { - panic!( - "expected a range partitioning, got {}", - repartition.partitioning() - ); - }; - assert_eq!(range.split_points().len(), 3); - assert!(range.ordering()[0].options.descending); + let plan = assemble(request).await.unwrap(); + let merge = plan + .downcast_ref::() + .expect("the stage's sorted partitions are merged"); + assert!(merge.expr()[0].options.descending); + // The stage reads the producer's four partitions without an exchange. + assert_eq!(merge.input().output_partitioning().partition_count(), 4); + let batches = collect(plan, SessionContext::new().task_ctx()) + .await + .unwrap(); + let values: Vec = batches + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_primitive::() + .values() + .to_vec() + }) + .collect(); + assert_eq!(values, (0..1000).rev().collect::>()); } -/// A range shuffle compares raw values, which order floats differently from Comet's sort. +/// A merge compares raw values, which order floats differently from Comet's sort. #[tokio::test(flavor = "multi_thread")] -async fn range_exchange_on_a_float_key_is_read_as_one_partition() { +async fn range_exchange_sorted_by_a_float_is_read_as_one_partition() { let mut producer = range_stage(1000, 4); producer.output_schema = vec![field(double_type())]; + let order = first_column_descending(double_type()); let request = query(vec![ producer, - reading_stage(0, range_partitioning(4, vec![first_column_descending()]), 4), + sorting_stage(0, range_partitioning(4, vec![]), order, double_type()), ]); - let (plan, samples) = assemble_counting_samples(request).await.unwrap(); - assert_eq!(samples, 0); - assert!(exchange_of(&plan).is::()); + let plan = assemble(request).await.unwrap(); + assert!(!plan.is::()); + assert!(plan.children()[1].is::()); + assert_eq!(plan.output_partitioning().partition_count(), 1); } #[tokio::test(flavor = "multi_thread")] diff --git a/native/native-executors/tests/cluster.rs b/native/native-executors/tests/cluster.rs index e38d5190ff8..681cff1d9a2 100644 --- a/native/native-executors/tests/cluster.rs +++ b/native/native-executors/tests/cluster.rs @@ -30,7 +30,7 @@ use std::time::{Duration, Instant, SystemTime}; use arrow_flight::error::FlightError; use arrow_flight::{FlightClient, FlightDescriptor, FlightEndpoint, FlightInfo}; use common::*; -use datafusion::arrow::array::RecordBatch; +use datafusion::arrow::array::{AsArray, RecordBatch}; use datafusion::arrow::datatypes::Int64Type; use datafusion_comet_proto::spark_native_executors::QueryRequest; use futures::TryStreamExt; @@ -207,34 +207,29 @@ async fn runs_a_query_across_a_range_exchange() { ); } -/// The scheduler samples the producer for split points, so each partition holds one range of -/// keys, in the exchange's order. +/// A global sort runs as a sort of each partition and a merge, so its result comes back as one +/// partition, in order. #[tokio::test(flavor = "multi_thread")] -async fn range_partitions_a_range_exchange_by_sampled_split_points() { +async fn sorts_across_a_range_exchange_by_merging_sorted_partitions() { let cluster = Cluster::start(); + let order = first_column_descending(long_type()); let request = query(vec![ range_stage(1000, 4), - reading_stage(0, range_partitioning(4, vec![first_column_descending()]), 4), + sorting_stage(0, range_partitioning(4, vec![]), order, long_type()), ]); let partitions = cluster.run(&request).await; - assert_eq!(partitions.len(), 4); - let ranges: Vec> = partitions + assert_eq!(partitions.len(), 1); + let values: Vec = partitions[0] .iter() - .map(|partition| sorted_values::(partition)) + .flat_map(|batch| { + batch + .column(0) + .as_primitive::() + .values() + .to_vec() + }) .collect(); - for (index, range) in ranges.iter().enumerate() { - assert!(!range.is_empty(), "partition {index} is empty"); - } - for pair in ranges.windows(2) { - assert!( - pair[0].first() > pair[1].last(), - "a partition holds keys below the next one's" - ); - } - assert_eq!( - sorted_values::(&partitions.concat()), - (0..1000).collect::>() - ); + assert_eq!(values, (0..1000).rev().collect::>()); } #[tokio::test(flavor = "multi_thread")] diff --git a/native/native-executors/tests/common/mod.rs b/native/native-executors/tests/common/mod.rs index 112a509581f..60d56ec5c3d 100644 --- a/native/native-executors/tests/common/mod.rs +++ b/native/native-executors/tests/common/mod.rs @@ -30,7 +30,7 @@ use datafusion_comet_proto::spark_expression::{ }; use datafusion_comet_proto::spark_native_executors::{QueryRequest, Stage, StageInput}; use datafusion_comet_proto::spark_operator::{ - operator::OpStruct, Operator, RangeScan, Scan, SparkStructField, + operator::OpStruct, Operator, RangeScan, Scan, Sort, SparkStructField, }; use datafusion_comet_proto::spark_partitioning::{ partitioning::PartitioningStruct, HashPartition, Partitioning, RangePartition, SinglePartition, @@ -131,6 +131,43 @@ pub fn reading_stage(producer: u32, partitioning: Partitioning, num_partitions: } } +/// A stage that sorts what stage `producer` sends it through an exchange with `partitioning`, by +/// `order`, of a column of type `data_type`, and declares that order, as a driver does for a +/// stage that reads a range exchange. +pub fn sorting_stage( + producer: u32, + partitioning: Partitioning, + order: Expr, + data_type: spark_expression::DataType, +) -> Stage { + let scan = Operator { + op_struct: Some(OpStruct::Scan(Scan { + fields: vec![data_type.clone()], + source: "test".to_string(), + })), + ..Default::default() + }; + Stage { + plan: Some(Operator { + children: vec![scan], + op_struct: Some(OpStruct::Sort(Sort { + sort_orders: vec![order.clone()], + ..Default::default() + })), + ..Default::default() + }), + inputs: vec![StageInput { + producer, + partitioning: Some(partitioning), + broadcast: false, + }], + num_partitions: 1, + output_schema: vec![field(data_type)], + output_ordering: vec![order], + ..Default::default() + } +} + pub fn hash_on_first_column(num_partitions: i32) -> Partitioning { Partitioning { partitioning_struct: Some(PartitioningStruct::HashPartition(HashPartition { @@ -157,11 +194,11 @@ pub fn range_partitioning(num_partitions: i32, sort_orders: Vec) -> Partit } } -/// The first column, descending, as a range exchange's sort key. -pub fn first_column_descending() -> Expr { +/// The first column, of type `data_type`, descending, as a sort key. +pub fn first_column_descending(data_type: spark_expression::DataType) -> Expr { Expr { expr_struct: Some(ExprStruct::SortOrder(Box::new(SortOrder { - child: Some(Box::new(bound(0, long_type()))), + child: Some(Box::new(bound(0, data_type))), direction: SortDirection::Descending as i32, null_ordering: NullOrdering::NullsLast as i32, }))), diff --git a/native/proto/src/proto/native_executors.proto b/native/proto/src/proto/native_executors.proto index 9c3ae718c59..9d084fe374b 100644 --- a/native/proto/src/proto/native_executors.proto +++ b/native/proto/src/proto/native_executors.proto @@ -23,6 +23,7 @@ syntax = "proto3"; package spark.spark_native_executors; +import "expr.proto"; import "operator.proto"; import "partitioning.proto"; @@ -58,6 +59,11 @@ message Stage { // leaves the stage, as a Spark task's shuffle writer does, because a hash exchange places // rows by the types of their values too. repeated spark.spark_operator.SparkStructField output_schema = 5; + // For a stage that reads a range exchange, the order its block sorts each partition by, as + // SortOrder expressions over the block's output. Rather than range-partition the exchange, + // which needs bounds Spark would sample for, the cluster reads the producer's partitions as + // they are, sorts each, and merges them in this order. + repeated spark.spark_expression.Expr output_ordering = 6; } // How a scheduler sends a stage to its executors, inside the task plans Ballista ships. The @@ -72,13 +78,6 @@ message Fragment { map conf = 4; } -// How a scheduler sends the sampling operator of a range exchange to its executors: a -// reservoir sample of up to `per_partition` rows of each partition, keeping the columns `keys`. -message Sample { - uint64 per_partition = 1; - repeated uint64 keys = 2; -} - message StageInput { // Index into QueryRequest.stages of the stage producing this input. uint32 producer = 1; diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala index 0874fb60cfc..15236b380fb 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala @@ -397,7 +397,16 @@ object CometNativeExecutors { val numPartitions = if (partitionPlans.nonEmpty) partitionPlans.length else block.outputPartitioning.numPartitions - addStage(plan, blockPlan, inputs, partitionPlans, numPartitions) + // A stage that reads a range exchange sorts each of its partitions. The cluster merges them + // in this order instead of range-partitioning the exchange. Without it, the cluster reads + // the exchange as one partition. + val outputOrdering = if (inputs.exists(_.getPartitioning.hasRangePartition)) { + val orders = block.outputOrdering.map(QueryPlanSerde.exprToProto(_, block.output)) + if (orders.nonEmpty && orders.forall(_.isDefined)) orders.flatten else Nil + } else { + Nil + } + addStage(plan, blockPlan, inputs, partitionPlans, numPartitions, outputOrdering) } /** An input of a stage that reads the stage running `producer`, as `configure` says. */ @@ -417,7 +426,7 @@ object CometNativeExecutors { val blockPlan = take .finalNativePlan(inputPartitions = 2) .getOrElse(fail(take, "its top-K plan cannot run natively")) - addStage(take, blockPlan, Seq(input), Seq.empty, 1) + addStage(take, blockPlan, Seq(input), Seq.empty, 1, Nil) } private def addStage( @@ -425,7 +434,8 @@ object CometNativeExecutors { blockPlan: Operator, inputs: Seq[StageInput], partitionPlans: Seq[ByteString], - numPartitions: Int): Int = { + numPartitions: Int, + outputOrdering: Seq[ExprOuterClass.Expr]): Int = { findJvmCallback(blockPlan).foreach { name => fail(plan, s"its native plan calls back into the JVM ($name)") } @@ -440,6 +450,7 @@ object CometNativeExecutors { .addAllPartitionPlans(partitionPlans.asJava) .setNumPartitions(numPartitions) .addAllOutputSchema(outputSchema.asJava) + .addAllOutputOrdering(outputOrdering.asJava) .build() val index = stages.length - 1 stageIndexes.put(plan, index) @@ -489,7 +500,7 @@ object CometNativeExecutors { .setNumPartitions(numPartitions)) .build() case RangePartitioning(ordering, numPartitions) => - // Without bounds: the cluster decides how to range-partition. + // Without bounds: the cluster merges the stage's sorted partitions instead. val sortOrders = ordering.map { order => QueryPlanSerde .exprToProto(order, exchange.child.output) From 48e5f9a5af418df85898282b65e9603479955e26 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 1 Nov 2026 07:36:57 -0700 Subject: [PATCH 25/36] docs: add a contributor guide page for Comet Native Executors with future hybrid execution research --- docs/source/contributor-guide/index.md | 1 + .../contributor-guide/native_executors.md | 109 ++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 docs/source/contributor-guide/native_executors.md diff --git a/docs/source/contributor-guide/index.md b/docs/source/contributor-guide/index.md index 53acfa1be69..350d61f7931 100644 --- a/docs/source/contributor-guide/index.md +++ b/docs/source/contributor-guide/index.md @@ -51,6 +51,7 @@ Floating-Point Semantics JVM Shuffle Native Shuffle Memory Management +Comet Native Executors Iceberg Writes ANSI Error Propagation S3 Credential Provider Design diff --git a/docs/source/contributor-guide/native_executors.md b/docs/source/contributor-guide/native_executors.md new file mode 100644 index 00000000000..991e3d95ecd --- /dev/null +++ b/docs/source/contributor-guide/native_executors.md @@ -0,0 +1,109 @@ + + +# Comet Native Executors + +**Experimental.** Comet Native Executors run whole Spark queries without a JVM in the data plane. A +Spark driver plans each query with Catalyst and Comet, then submits the plan as a stage graph of +Comet native blocks to a cluster of one `comet-scheduler` and any number of `comet-executor` +processes, built on [Apache DataFusion Ballista](https://github.com/apache/datafusion-ballista). +The design is in [issue #7288](https://github.com/apache/datafusion-comet/issues/7288), and +`native/native-executors/README.md` describes the cluster, how to build it and how to test it. + +## Current limitation: whole queries only + +The driver offloads a query only if every operator and expression in it can run natively +(`CometNativeExecutors` in `CometNativeExecutorsExec.scala`). Otherwise it refuses the query, +naming the operator that couldn't run and Comet's fallback reasons for it. That includes plans that +call back into the JVM, such as JVM scalar UDFs and Arrow Python UDFs, because the cluster has no +JVM. + +Data already crosses between Spark and the cluster in two places: + +- **Scalar subqueries.** Spark runs each scalar subquery, and the driver writes its result into the + plan as a literal before submitting the query. Data flows from Spark to the cluster, but only one + value. +- **Query results.** The driver reads the final stage's output over Arrow Flight, one endpoint per + result partition (`ResultEndpoint`). Data flows from the cluster to Spark, but only at the root + of the plan. + +## Possible future research: hybrid Spark and native execution + +A query could run partly on the cluster and partly in Spark, so that a query with one unsupported +operator or UDF can still run most of its work natively. The approaches below generalize the two +crossings above. They differ mainly in which way data crosses the boundary. Moving data from the +cluster into Spark is much easier than moving data from Spark into the cluster. + +### A native subtree as a leaf of the Spark plan + +The driver could offload the largest native subtrees under an operator that must run in Spark, +rather than requiring the whole plan to be native. Spark would see each offloaded subtree as a leaf +with one partition per result endpoint, and each Spark task would read its partition from the +cluster over Flight. The batches arrive as Arrow, so Comet's columnar operators above the leaf can +consume them without converting them to rows. + +This reuses the result path that already works. What remains: + +- Serving partitioned results in pipelined mode. Today the scheduler coalesces a pipelined job's + results to one partition. +- Reporting the cluster's output statistics to Spark's adaptive query execution. +- Choosing where the native region should end. + +It suits the common shape where the scans, joins and partial aggregates are native, and an +unsupported operator or UDF sits near the top of the plan. + +### Small Spark outputs as inputs to native stages + +Spark could compute a whole relation that fits in a broadcast, such as the build side of a +broadcast join, rather than a single value. The driver would encode the relation as Arrow IPC and +send it with the plan or upload it to the scheduler, where a native leaf reads it. This only +works for small data, but it costs little, because it uses the same round trip through the driver +that scalar subqueries already make. + +### A shuffle both engines can read and write + +Mixing stages freely needs a shuffle that Spark tasks can write and cluster tasks can read, and the +other way round. Some options: + +- **A remote shuffle service.** Comet already integrates with Apache Celeborn. A Celeborn shuffle + reader and writer for Ballista would let either engine run any stage. +- **An object store or shared filesystem.** Comet's native shuffle already writes Arrow IPC. Spark + stages would write it to a location the cluster can read, and the scheduler would register those + files as a finished stage's output. +- **Pushing batches over Flight.** Spark tasks would send batches to executors with `DoPut`. This is + simpler to deploy, but it ties the lifetimes of Spark tasks and cluster tasks together and makes + retrying a failed task difficult. + +The hard part is scheduling rather than the data format. Spark's DAG scheduler and Ballista's +scheduler would have to agree on stage dependencies, failure handling and adaptive replanning. +Realistically, Spark stays the coordinator, and the cluster runs individual stages on its behalf. + +### Running JVM expressions on native executors + +Instead of splitting the plan, `comet-executor` could embed a JVM to evaluate the expressions and +UDFs that have no native implementation, reusing Comet's existing bridges for calling back into +the JVM. The plan would stay fully native and scheduling would not change. The costs are that every +executor needs Spark's classpath and enough session state to evaluate the expressions, and that the +plan must carry those expressions in serialized Spark form. + +### When splitting pays off + +Whichever approach is used, the driver should decide whether to split a query based on cost. If +most of a query's work has to run in Spark, running all of it with Comet on Spark is likely to beat +sending part of it to the cluster and reading the results back over the network. From 6a83a2c2dcdfceecf0bfb7f7a4ff72b66df341f7 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 1 Nov 2026 07:48:44 -0700 Subject: [PATCH 26/36] docs: describe Spark protocol compatibility as future research for Comet Native Executors --- .../contributor-guide/native_executors.md | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/docs/source/contributor-guide/native_executors.md b/docs/source/contributor-guide/native_executors.md index 991e3d95ecd..83d4b5a8877 100644 --- a/docs/source/contributor-guide/native_executors.md +++ b/docs/source/contributor-guide/native_executors.md @@ -102,6 +102,62 @@ the JVM. The plan would stay fully native and scheduling would not change. The c executor needs Spark's classpath and enough session state to evaluate the expressions, and that the plan must carry those expressions in serialized Spark form. +### Speaking Spark's protocols + +Another approach is for `comet-executor` to speak Spark's own protocols, so that Spark tasks and +native tasks exchange data the way Spark executors do. A Spark executor uses two protocols, and +they differ in how practical they are to implement. + +The control plane runs over Spark's Netty RPC. Its messages, such as `RegisterExecutor`, +`LaunchTask`, `StatusUpdate` and heartbeats, are Scala case classes in Java serialization. A +`LaunchTask` carries a serialized `ShuffleMapTask` or `ResultTask`, whose RDD chain and closures +are JVM code, and the formats change between Spark versions. + +The data plane is Spark's block transfer protocol in `network-common` (`OpenBlocks`, +`FetchShuffleBlocks`, `StreamHandle`, `ChunkFetchRequest` and so on). Its messages have a compact +binary encoding rather than Java serialization, the protocol has been stable for many releases, and +the external shuffle service speaks it too. Shuffle data sits in Spark's sort-based layout of a data +file and an index file per map task. Comet's native shuffle already writes that layout: the data +comes from Rust and `CometNativeShuffleWriter` commits the index through Spark's +`IndexShuffleBlockResolver`. + +That suggests three options: + +- **A fully native Spark executor.** Not practical. Running a Spark task means deserializing Java + objects and running JVM closures. Even a task that only wraps a Comet plan would need its plan + extracted from a Java-serialized object graph, separately for each Spark version. +- **A native data plane, with Spark coordinating.** `comet-executor` writes shuffle output in + Spark's layout, serves those blocks over the block transfer protocol, and fetches blocks from + Spark executors and the external shuffle service. When a native stage finishes, the driver + registers its map outputs with Spark's `MapOutputTrackerMaster`, and downstream Spark stages read + them as they would any other shuffle. In the other direction, the driver gives a native stage the + block locations of the Spark shuffle it reads. This provides a shuffle both engines can read and + write without a separate shuffle service. The work includes Spark's authentication and encryption + (SASL, or `spark.network.crypto`), which a first version could require to be off, mapping fetch + failures onto Spark's `FetchFailed` handling and stage retries, and possibly push-based shuffle + later. A native block fetch client would also benefit Comet on Spark, by moving shuffle reads out + of the JVM. +- **An embedded JVM for the control plane.** `comet-executor` hosts a small JVM that runs a cut-down + `CoarseGrainedExecutorBackend`. It registers with the driver, deserializes tasks, passes each + task's Comet plan to native code, and reports map statuses and accumulators. This works, but the + result is a Spark executor with a small heap whose tasks run natively: Comet on Spark with a more + native data plane. Spark schedules every task one stage at a time, so the cluster's pipelined + execution and lower per-task overhead are lost. An embedded JVM is more useful as an evaluator for + expressions that have no native implementation, as described above, where Spark's protocols are + not involved. + +The data-plane option combines with either way of placing native work. Native executors could join +a Spark application as executors, which needs the embedded JVM for their control plane, with +stage-level scheduling sending fully native stages to them through a `ResourceProfile` that asks for +a resource only native executors offer. Or the driver could keep offloading whole native subgraphs +to the cluster, which keeps its own scheduling and pipelining, and use Spark-compatible shuffle as +the boundary between Spark stages and native stages. The second keeps more of what the cluster +provides. + +A first experiment is a Rust client and server for `OpenBlocks` and `FetchShuffleBlocks`, tested +against a Spark executor and the external shuffle service by reading a shuffle that Comet's native +shuffle wrote. That shows whether the data plane works independently of any scheduling decision. + ### When splitting pays off Whichever approach is used, the driver should decide whether to split a query based on cost. If From 255a10c78df83121decf43bb5677aa1a1644a661 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 1 Nov 2026 07:49:02 -0700 Subject: [PATCH 27/36] docs: link the block transfer experiment issue --- docs/source/contributor-guide/native_executors.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/contributor-guide/native_executors.md b/docs/source/contributor-guide/native_executors.md index 83d4b5a8877..d81be8fb0fd 100644 --- a/docs/source/contributor-guide/native_executors.md +++ b/docs/source/contributor-guide/native_executors.md @@ -154,7 +154,7 @@ to the cluster, which keeps its own scheduling and pipelining, and use Spark-com the boundary between Spark stages and native stages. The second keeps more of what the cluster provides. -A first experiment is a Rust client and server for `OpenBlocks` and `FetchShuffleBlocks`, tested +A first experiment ([#7384](https://github.com/apache/datafusion-comet/issues/7384)) is a Rust client and server for `OpenBlocks` and `FetchShuffleBlocks`, tested against a Spark executor and the external shuffle service by reading a shuffle that Comet's native shuffle wrote. That shows whether the data plane works independently of any scheduling decision. From e9d6e3da06a2244a84590e56f9b204a74576d022 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 1 Nov 2026 12:43:53 -0700 Subject: [PATCH 28/36] feat: write the scheduler's event logs to COMET_SCHEDULER_EVENT_LOG_DIR --- native/native-executors/src/bin/comet-scheduler.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/native/native-executors/src/bin/comet-scheduler.rs b/native/native-executors/src/bin/comet-scheduler.rs index 01b51b96c74..8f078017bf2 100644 --- a/native/native-executors/src/bin/comet-scheduler.rs +++ b/native/native-executors/src/bin/comet-scheduler.rs @@ -30,6 +30,9 @@ //! accepts, such as a task with many partitions of a scan (default: Ballista's 16 MiB) //! - `COMET_SCHEDULER_BALLISTA_CONFIG`: Ballista settings for every query, as comma-separated //! `key=value` pairs, such as `ballista.execution.mode=pipelined` +//! - `COMET_SCHEDULER_EVENT_LOG_DIR`: a directory to write one event log per query to, as +//! Ballista's scheduler does: the query's plan, a record for each task, and each stage's rows +//! and compute time (default: no event logs) use std::net::SocketAddr; use std::sync::Arc; @@ -68,6 +71,9 @@ async fn main() -> Result<(), Box> { if let Some(seconds) = env_var("COMET_SCHEDULER_RESULT_RETENTION_SECONDS")? { config.finished_job_data_clean_up_interval_seconds = seconds; } + if let Some(dir) = env_var::("COMET_SCHEDULER_EVENT_LOG_DIR")? { + config.event_log_dir = Some(dir); + } let flight_port: u16 = env_var("COMET_SCHEDULER_FLIGHT_PORT")?.unwrap_or(50060); let ballista_settings = parse_settings(&env_var::("COMET_SCHEDULER_BALLISTA_CONFIG")?.unwrap_or_default())?; From 4b1d030d988dffe1bb7643919c07e09527f58a43 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 2 Nov 2026 06:37:23 -0700 Subject: [PATCH 29/36] feat: broadcast a small build side into the probe side's stage on Comet Native Executors Spark places an exchange on both sides of a shuffled join, so when the build side turns out small a broadcast alone still shuffles the whole probe side. Comet's adaptive rules, added to Ballista's adaptive planning, hold back the probe side's stage where Spark's estimates say the build side is worth measuring, then once the build side's stage finishes either move the join into the probe side's stage with the build side broadcast to it, or release the probe side as it was. The driver sends Spark's size estimate of each stage for the decision. Prototype for #7391, built against Ballista's comet-native branch plus a hook for adaptive rules on a fork. --- .../execution/native_executors/fragment.rs | 24 + .../src/execution/native_executors/mod.rs | 4 +- native/native-executors/Cargo.lock | 6 +- native/native-executors/Cargo.toml | 9 +- native/native-executors/src/adaptive.rs | 499 ++++++++++++++++++ native/native-executors/src/assembly.rs | 1 + .../src/bin/comet-scheduler.rs | 2 + native/native-executors/src/lib.rs | 1 + native/native-executors/tests/cluster.rs | 78 ++- native/native-executors/tests/common/mod.rs | 38 +- native/proto/src/proto/native_executors.proto | 5 + .../sql/comet/CometNativeExecutorsExec.scala | 18 +- 12 files changed, 673 insertions(+), 12 deletions(-) create mode 100644 native/native-executors/src/adaptive.rs diff --git a/native/core/src/execution/native_executors/fragment.rs b/native/core/src/execution/native_executors/fragment.rs index afa10645955..7814ae2c3d8 100644 --- a/native/core/src/execution/native_executors/fragment.rs +++ b/native/core/src/execution/native_executors/fragment.rs @@ -24,6 +24,7 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef, UInt32Type}; use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{internal_datafusion_err, DataFusionError, Result}; use datafusion::datasource::memory::MemorySourceConfig; +use datafusion::datasource::source::DataSourceExec; use datafusion::execution::disk_manager::DiskManager; use datafusion::execution::memory_pool::{MemoryPool, UnboundedMemoryPool}; use datafusion::execution::TaskContext; @@ -88,6 +89,29 @@ pub fn partition_descriptor( Ok(MemorySourceConfig::try_new_exec(&partitions, schema, None)?) } +/// The partitions' own copies of the block in a fragment's partition `descriptor`, one per +/// partition, or none when its partitions all run the fragment's block. +pub fn partition_plans(descriptor: &Arc) -> Result>> { + let memory = descriptor + .downcast_ref::() + .and_then(|source| source.data_source().downcast_ref::()) + .ok_or_else(|| internal_datafusion_err!("Not a fragment's partition descriptor"))?; + let mut plans = Vec::new(); + for partition in memory.partitions() { + let [batch] = partition.as_slice() else { + return Err(internal_datafusion_err!( + "A fragment's partition descriptor must have one row per partition" + )); + }; + let column = batch.column(1).as_binary::(); + if column.is_null(0) { + return Ok(vec![]); + } + plans.push(column.value(0).to_vec()); + } + Ok(plans) +} + /// Runs a Comet native block for each partition of a stage. /// /// The first child is the partition descriptor (see [`partition_descriptor`]), and each other diff --git a/native/core/src/execution/native_executors/mod.rs b/native/core/src/execution/native_executors/mod.rs index 48c1405bad5..9201fd652b4 100644 --- a/native/core/src/execution/native_executors/mod.rs +++ b/native/core/src/execution/native_executors/mod.rs @@ -28,7 +28,9 @@ mod client; mod fragment; -pub use fragment::{block_schema, fragment_schema, partition_descriptor, CometFragmentExec}; +pub use fragment::{ + block_schema, fragment_schema, partition_descriptor, partition_plans, CometFragmentExec, +}; /// The version of this Comet build. A Comet Native Executor cluster requires a driver's to match /// its own, together with [`datafusion_comet_proto::PROTO_FINGERPRINT`]. diff --git a/native/native-executors/Cargo.lock b/native/native-executors/Cargo.lock index e8fd77bcb92..cb87cfa8597 100644 --- a/native/native-executors/Cargo.lock +++ b/native/native-executors/Cargo.lock @@ -999,7 +999,7 @@ dependencies = [ [[package]] name = "ballista-core" version = "55.0.0" -source = "git+https://github.com/apache/datafusion-ballista?branch=comet-native#8ab06362af7d74c01be6d9ceb16a8cad8efc8b7c" +source = "git+https://github.com/andygrove/datafusion-ballista?branch=comet-adaptive-rules#56e2d8a2c6c387f7cbd856950b7814777aa31f51" dependencies = [ "arrow-flight", "async-trait", @@ -1028,7 +1028,7 @@ dependencies = [ [[package]] name = "ballista-executor" version = "55.0.0" -source = "git+https://github.com/apache/datafusion-ballista?branch=comet-native#8ab06362af7d74c01be6d9ceb16a8cad8efc8b7c" +source = "git+https://github.com/andygrove/datafusion-ballista?branch=comet-adaptive-rules#56e2d8a2c6c387f7cbd856950b7814777aa31f51" dependencies = [ "arrow", "arrow-flight", @@ -1057,7 +1057,7 @@ dependencies = [ [[package]] name = "ballista-scheduler" version = "55.0.0" -source = "git+https://github.com/apache/datafusion-ballista?branch=comet-native#8ab06362af7d74c01be6d9ceb16a8cad8efc8b7c" +source = "git+https://github.com/andygrove/datafusion-ballista?branch=comet-adaptive-rules#56e2d8a2c6c387f7cbd856950b7814777aa31f51" dependencies = [ "arrow-flight", "async-trait", diff --git a/native/native-executors/Cargo.toml b/native/native-executors/Cargo.toml index f83d7536256..0d77383bc46 100644 --- a/native/native-executors/Cargo.toml +++ b/native/native-executors/Cargo.toml @@ -23,6 +23,9 @@ # branch, which is Ballista's main branch with the enhancements Comet Native Executors need that # haven't merged yet (apache/datafusion-ballista#2682, #2721, #2722 and #2723), on the same # DataFusion major version. +# +# Prototype for apache/datafusion-comet#7391: builds against `comet-adaptive-rules` on a fork, +# which is `comet-native` plus a hook for adaptive planning rules of Comet's own. [workspace] [package] @@ -37,9 +40,9 @@ publish = false [dependencies] arrow-flight = "59.2.0" -ballista-core = { git = "https://github.com/apache/datafusion-ballista", branch = "comet-native" } -ballista-executor = { git = "https://github.com/apache/datafusion-ballista", branch = "comet-native", default-features = false, features = ["arrow-ipc-optimizations"] } -ballista-scheduler = { git = "https://github.com/apache/datafusion-ballista", branch = "comet-native", default-features = false } +ballista-core = { git = "https://github.com/andygrove/datafusion-ballista", branch = "comet-adaptive-rules" } +ballista-executor = { git = "https://github.com/andygrove/datafusion-ballista", branch = "comet-adaptive-rules", default-features = false, features = ["arrow-ipc-optimizations"] } +ballista-scheduler = { git = "https://github.com/andygrove/datafusion-ballista", branch = "comet-adaptive-rules", default-features = false } datafusion = { version = "55.2.0", default-features = false } # Comet's core crate, under the name of its library. Without the default hdfs-opendal feature, # which links libjvm through libhdfs. diff --git a/native/native-executors/src/adaptive.rs b/native/native-executors/src/adaptive.rs new file mode 100644 index 00000000000..b93b606664c --- /dev/null +++ b/native/native-executors/src/adaptive.rs @@ -0,0 +1,499 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Adaptive join planning for Comet fragments. +//! +//! A stage's joins are inside its Comet block, so Ballista's adaptive join rules see the stage +//! as a leaf, and Spark has already placed an exchange on each side of every shuffled join. When +//! a join's build side turns out small, broadcasting it is not enough: the probe side would still +//! be shuffled whole to reach the join. These rules move the join into the stage that produces +//! the probe side instead, with the build side broadcast to it, so the probe side's exchange +//! carries the join's output rather than all of its input. +//! +//! That only pays off if the probe side's stage hasn't run yet when the build side's size is +//! known. [`HoldProbeStages`] holds it back before anything runs, where Spark's estimates say the +//! measurement could change the plan, and [`BroadcastIntoProbe`] decides once the build side's +//! stage finishes: it moves the join where the build side measured small, and otherwise releases +//! the probe side's stage as it was. Holding serialises the two stages, so like Ballista's own +//! build-side staging it uses Ballista's broadcast threshold, minimum probe ratio and maximum +//! estimate multiple. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use ballista_core::config::BallistaConfig; +use ballista_scheduler::state::adaptive::{AdaptiveRule, AdaptiveRules, ExchangeExec}; +use comet::execution::native_executors::{ + partition_descriptor, partition_plans, CometFragmentExec, +}; +use datafusion::common::tree_node::{Transformed, TreeNode}; +use datafusion::common::{internal_datafusion_err, Result}; +use datafusion::config::ConfigOptions; +use datafusion::physical_optimizer::PhysicalOptimizerRule; +use datafusion::physical_plan::statistics::StatisticsContext; +use datafusion::physical_plan::{ + ChildrenPropertiesMode, ExecutionPlan, Partitioning, ReplaceChildrenOptions, StatisticsArgs, +}; +use datafusion_comet_proto::spark_expression::expr::ExprStruct; +use datafusion_comet_proto::spark_native_executors::Fragment; +use datafusion_comet_proto::spark_operator::operator::OpStruct; +use datafusion_comet_proto::spark_operator::{BuildSide, JoinType, Operator, ShuffleScan}; +use log::info; +use prost::Message; + +/// The rules Comet adds to Ballista's adaptive planning. +#[derive(Debug, Default)] +pub struct CometAdaptiveRules; + +impl AdaptiveRules for CometAdaptiveRules { + fn preparation_rules(&self, _plan_ids: Arc) -> Vec { + vec![Arc::new(HoldProbeStages)] + } + + fn replan_rules(&self, plan_ids: Arc) -> Vec { + vec![Arc::new(BroadcastIntoProbe { plan_ids })] + } +} + +/// A join in a stage's block that can run in its probe side's stage instead. +/// +/// It is an inner hash join built on the right, with no condition and with keys that are +/// columns, whose inputs are both input leaves of the block. Moved below the probe side's +/// exchange its output keeps the probe side's columns first, so the exchange's keys still name +/// the same columns. +struct MovableJoin { + /// The child indexes from the block's root to the join. + path: Vec, + /// The stage inputs that the join's probe (left) and build (right) leaves read. + probe_input: usize, + build_input: usize, +} + +/// The joins of `block` that [`MovableJoin`] describes. +fn movable_joins(block: &Operator) -> Vec { + fn walk( + op: &Operator, + path: &mut Vec, + leaves: &mut usize, + found: &mut Vec, + ) { + if is_input_leaf(op) { + *leaves += 1; + return; + } + if let Some(OpStruct::HashJoin(join)) = &op.op_struct { + let movable = join.join_type == JoinType::Inner as i32 + && join.build_side == BuildSide::BuildRight as i32 + && join.condition.is_none() + && !join.null_aware_anti_join + && join + .left_join_keys + .iter() + .chain(&join.right_join_keys) + .all(|key| matches!(key.expr_struct, Some(ExprStruct::Bound(_)))) + && op.children.len() == 2 + && op.children.iter().all(is_input_leaf); + if movable { + found.push(MovableJoin { + path: path.clone(), + probe_input: *leaves, + build_input: *leaves + 1, + }); + } + } + for (index, child) in op.children.iter().enumerate() { + path.push(index); + walk(child, path, leaves, found); + path.pop(); + } + } + let mut found = vec![]; + walk(block, &mut vec![], &mut 0, &mut found); + found +} + +/// Whether `op` reads one of its stage's inputs. Comet's planner feeds them to these leaves in +/// the order a depth-first walk meets them. +fn is_input_leaf(op: &Operator) -> bool { + matches!( + op.op_struct, + Some(OpStruct::Scan(_)) | Some(OpStruct::ShuffleScan(_)) + ) +} + +fn leaf_fields(op: &Operator) -> &[datafusion_comet_proto::spark_expression::DataType] { + match &op.op_struct { + Some(OpStruct::Scan(scan)) => &scan.fields, + Some(OpStruct::ShuffleScan(scan)) => &scan.fields, + _ => &[], + } +} + +fn node_at<'a>(block: &'a Operator, path: &[usize]) -> &'a Operator { + path.iter().fold(block, |op, index| &op.children[*index]) +} + +/// A stage's block, or `None` for a stage that reads files, whose partitions each run their own +/// copy of it. +fn block_of(fragment: &CometFragmentExec) -> Result> { + let block = &fragment.fragment().block; + if block.is_empty() { + return Ok(None); + } + Ok(Some(Operator::decode(block.as_slice()).map_err(|e| { + internal_datafusion_err!("Cannot decode a stage's block: {e}") + })?)) +} + +/// A stage input's exchange, when it is a hash shuffle of another stage's output. +fn hash_exchange(input: &Arc) -> Option<(&ExchangeExec, &CometFragmentExec)> { + let exchange = input.downcast_ref::()?; + if exchange.is_broadcast() || !matches!(exchange.partitioning(), Some(Partitioning::Hash(..))) { + return None; + } + let producer = exchange.input().downcast_ref::()?; + Some((exchange, producer)) +} + +fn ballista_config(config: &ConfigOptions) -> BallistaConfig { + config + .extensions + .get::() + .cloned() + .unwrap_or_default() +} + +/// Holds back the probe side's stage of each [`MovableJoin`] whose build side Spark estimates +/// close enough to the broadcast threshold that measuring it could make it a broadcast, and +/// whose probe side Spark estimates large enough that not shuffling it would pay for the wait. +/// +/// Spark's estimate of a filtered relation ignores the filter, so a build side under a selective +/// filter is often far smaller than its estimate. +#[derive(Debug)] +pub struct HoldProbeStages; + +impl PhysicalOptimizerRule for HoldProbeStages { + fn optimize( + &self, + plan: Arc, + config: &ConfigOptions, + ) -> Result> { + let config = ballista_config(config); + let threshold = config.broadcast_join_threshold_bytes() as u64; + if threshold == 0 || !config.adaptive_join_enabled() { + return Ok(plan); + } + let max_build = + threshold.saturating_mul(config.stage_build_side_max_estimate_multiple() as u64); + let min_probe_ratio = config.stage_build_side_min_probe_ratio() as u64; + Ok(plan + .transform_up(|node| { + let Some(fragment) = node.downcast_ref::() else { + return Ok(Transformed::no(node)); + }; + let Some(block) = block_of(fragment)? else { + return Ok(Transformed::no(node)); + }; + let inputs = &fragment.children()[1..]; + let mut held = inputs + .iter() + .map(|input| Arc::clone(*input)) + .collect::>(); + let mut changed = false; + for join in movable_joins(&block) { + let (Some((probe, probe_stage)), Some((build, build_stage))) = ( + hash_exchange(inputs[join.probe_input]), + hash_exchange(inputs[join.build_input]), + ) else { + continue; + }; + let (Some(probe_bytes), Some(build_bytes)) = ( + probe_stage.fragment().estimated_output_bytes, + build_stage.fragment().estimated_output_bytes, + ) else { + continue; + }; + if probe.shuffle_created() + || build.shuffle_created() + || probe.is_inactive_stage() + || build_bytes > max_build + || probe_bytes < build_bytes.saturating_mul(min_probe_ratio) + { + continue; + } + info!( + "Holding back a join's probe side (estimated {probe_bytes} bytes) until \ + its build side (estimated {build_bytes} bytes) is measured" + ); + held[join.probe_input] = Arc::new(probe.with_inactive_stage(true)); + changed = true; + } + if !changed { + return Ok(Transformed::no(node)); + } + let children = std::iter::once(Arc::clone(fragment.children()[0])) + .chain(held) + .collect(); + Ok(Transformed::yes(with_children(&node, children)?)) + })? + .data) + } + + fn name(&self) -> &str { + "HoldProbeStages" + } + + fn schema_check(&self) -> bool { + true + } +} + +/// Decides each join whose probe side [`HoldProbeStages`] held back, once its build side's stage +/// has finished: a build side under the broadcast threshold is broadcast to the probe side's +/// stage, which runs the join, and otherwise the probe side's stage is released unchanged. +#[derive(Debug)] +pub struct BroadcastIntoProbe { + plan_ids: Arc, +} + +impl PhysicalOptimizerRule for BroadcastIntoProbe { + fn optimize( + &self, + plan: Arc, + config: &ConfigOptions, + ) -> Result> { + let config = ballista_config(config); + let threshold_bytes = config.broadcast_join_threshold_bytes(); + let threshold_rows = config.broadcast_join_threshold_rows(); + Ok(plan + .transform_up(|node| { + let mut node = node; + let mut changed = false; + // Each decision changes the block, so find the joins again after each. + while let Some(decided) = self.decide_one(&node, threshold_bytes, threshold_rows)? { + node = decided; + changed = true; + } + Ok(if changed { + Transformed::yes(node) + } else { + Transformed::no(node) + }) + })? + .data) + } + + fn name(&self) -> &str { + "BroadcastIntoProbe" + } + + fn schema_check(&self) -> bool { + true + } +} + +impl BroadcastIntoProbe { + /// `node` with one held join decided, or `None` when it has none whose build side is ready. + fn decide_one( + &self, + node: &Arc, + threshold_bytes: usize, + threshold_rows: usize, + ) -> Result>> { + let Some(fragment) = node.downcast_ref::() else { + return Ok(None); + }; + let Some(block) = block_of(fragment)? else { + return Ok(None); + }; + let inputs = &fragment.children()[1..]; + for join in movable_joins(&block) { + let (Some((probe, _)), Some((build, _))) = ( + hash_exchange(inputs[join.probe_input]), + hash_exchange(inputs[join.build_input]), + ) else { + continue; + }; + if !probe.is_inactive_stage() || !build.shuffle_created() { + continue; + } + let stats = StatisticsContext::new().compute(build, &StatisticsArgs::new())?; + let small = match ( + stats.total_byte_size.get_value(), + stats.num_rows.get_value(), + ) { + (Some(bytes), _) => *bytes <= threshold_bytes, + (None, Some(rows)) => *rows <= threshold_rows, + (None, None) => false, + }; + info!( + "A held join's build side measured {:?} bytes and {:?} rows: {}", + stats.total_byte_size, + stats.num_rows, + if small { + "broadcasting it to the probe side's stage" + } else { + "shuffling the probe side" + } + ); + let decided = if small { + self.broadcast_into_probe(fragment, &block, &join)? + } else { + let mut children = fragment + .children() + .into_iter() + .map(Arc::clone) + .collect::>(); + children[1 + join.probe_input] = Arc::new(probe.with_inactive_stage(false)); + with_children(node, children)? + }; + return Ok(Some(decided)); + } + Ok(None) + } + + /// `consumer` with `join` moved into the stage that produces its probe side, and its build + /// side broadcast to that stage. + fn broadcast_into_probe( + &self, + consumer: &CometFragmentExec, + block: &Operator, + join: &MovableJoin, + ) -> Result> { + let inputs = &consumer.children()[1..]; + let (_, probe_stage) = hash_exchange(inputs[join.probe_input]) + .ok_or_else(|| internal_datafusion_err!("The join's probe side is not an exchange"))?; + let (build, build_stage) = hash_exchange(inputs[join.build_input]) + .ok_or_else(|| internal_datafusion_err!("The join's build side is not an exchange"))?; + let join_op = node_at(block, &join.path); + let build_leaf = &join_op.children[1]; + + // The probe side's stage runs the join on its output, reading the build side whole as + // its last input. The join becomes the root of the stage's block, so it takes over the + // block's pool of SQL texts. + let into_probe = |root: Operator| -> Operator { + let mut root = root; + let mut moved = join_op.clone(); + moved.sql_text_pool = std::mem::take(&mut root.sql_text_pool); + moved.children = vec![root, build_leaf.clone()]; + moved + }; + let probe_fragment = probe_stage.fragment(); + let block_of_probe = match block_of(probe_stage)? { + Some(root) => into_probe(root).encode_to_vec(), + None => vec![], + }; + let descriptor = Arc::clone(probe_stage.children()[0]); + let own_plans = partition_plans(&descriptor)?; + let descriptor = if own_plans.is_empty() { + descriptor + } else { + let plans = own_plans + .iter() + .map(|plan| { + let root = Operator::decode(plan.as_slice()).map_err(|e| { + internal_datafusion_err!("Cannot decode a partition's block: {e}") + })?; + Ok(into_probe(root).encode_to_vec()) + }) + .collect::>>()?; + partition_descriptor(plans.len(), &plans)? + }; + let mut broadcast = probe_fragment.broadcast.clone(); + broadcast.push(true); + let mut output_schema = probe_fragment.output_schema.clone(); + output_schema.extend(build_stage.fragment().output_schema.iter().cloned()); + let joined = Fragment { + block: block_of_probe, + output_schema, + broadcast, + conf: probe_fragment.conf.clone(), + estimated_output_bytes: None, + }; + let mut children = vec![descriptor]; + children.extend( + probe_stage.children()[1..] + .iter() + .map(|child| Arc::clone(*child)), + ); + children.push(Arc::new( + build.to_broadcast(self.plan_ids.fetch_add(1, Ordering::Relaxed)), + )); + let joined: Arc = + Arc::new(CometFragmentExec::try_new(Arc::new(joined), children)?); + let exchange = with_children(inputs[join.probe_input], vec![joined])?; + let exchange = exchange + .downcast_ref::() + .ok_or_else(|| internal_datafusion_err!("An exchange rebuilt as something else"))? + .with_inactive_stage(false); + + // The consumer reads the join's output where it read the join's probe side, and no + // longer reads the build side. + let mut fields = leaf_fields(&join_op.children[0]).to_vec(); + fields.extend(leaf_fields(build_leaf).iter().cloned()); + let leaf = Operator { + children: vec![], + plan_id: join_op.plan_id, + sql_text_pool: if join.path.is_empty() { + join_op.sql_text_pool.clone() + } else { + vec![] + }, + op_struct: Some(OpStruct::ShuffleScan(ShuffleScan { + fields, + source: "a join moved into the probe side's stage".to_string(), + })), + }; + let mut consumer_block = block.clone(); + replace_at(&mut consumer_block, &join.path, leaf); + let mut consumer_broadcast = consumer.fragment().broadcast.clone(); + consumer_broadcast.remove(join.build_input); + let consumer_fragment = Fragment { + block: consumer_block.encode_to_vec(), + broadcast: consumer_broadcast, + ..consumer.fragment().as_ref().clone() + }; + let mut children = consumer + .children() + .into_iter() + .map(Arc::clone) + .collect::>(); + children[1 + join.probe_input] = Arc::new(exchange); + children.remove(1 + join.build_input); + Ok(Arc::new(CometFragmentExec::try_new( + Arc::new(consumer_fragment), + children, + )?)) + } +} + +fn replace_at(block: &mut Operator, path: &[usize], replacement: Operator) { + let target = path + .iter() + .fold(block, |op, index| &mut op.children[*index]); + *target = replacement; +} + +fn with_children( + plan: &Arc, + children: Vec>, +) -> Result> { + Arc::clone(plan).replace_children( + children, + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + ) +} diff --git a/native/native-executors/src/assembly.rs b/native/native-executors/src/assembly.rs index 5b1e1fde60b..89f79d2d7d6 100644 --- a/native/native-executors/src/assembly.rs +++ b/native/native-executors/src/assembly.rs @@ -169,6 +169,7 @@ fn build_stage( broadcast: stage.inputs.iter().map(|input| input.broadcast).collect(), output_schema: stage.output_schema, conf: conf.clone(), + estimated_output_bytes: stage.estimated_output_bytes, }; let fragment: Arc = Arc::new(CometFragmentExec::try_new(Arc::new(fragment), children)?); diff --git a/native/native-executors/src/bin/comet-scheduler.rs b/native/native-executors/src/bin/comet-scheduler.rs index 8f078017bf2..f5c53a7489f 100644 --- a/native/native-executors/src/bin/comet-scheduler.rs +++ b/native/native-executors/src/bin/comet-scheduler.rs @@ -41,6 +41,7 @@ use arrow_flight::flight_service_server::FlightServiceServer; use ballista_scheduler::cluster::BallistaCluster; use ballista_scheduler::config::SchedulerConfig; use ballista_scheduler::scheduler_process::{create_scheduler, start_grpc_service}; +use comet_native_executors::adaptive::CometAdaptiveRules; use comet_native_executors::codec::CometPhysicalCodec; use comet_native_executors::frontend::CometFlightService; use comet_native_executors::{env_var, init_logging}; @@ -52,6 +53,7 @@ async fn main() -> Result<(), Box> { init_logging(); let mut config = SchedulerConfig { override_physical_codec: Some(Arc::new(CometPhysicalCodec::default())), + adaptive_rules: Some(Arc::new(CometAdaptiveRules)), ..Default::default() }; if let Some(host) = env_var("COMET_SCHEDULER_BIND_HOST")? { diff --git a/native/native-executors/src/lib.rs b/native/native-executors/src/lib.rs index 02251c83cb2..2457cf22cc7 100644 --- a/native/native-executors/src/lib.rs +++ b/native/native-executors/src/lib.rs @@ -28,6 +28,7 @@ use std::fmt::Display; use std::str::FromStr; +pub mod adaptive; pub mod assembly; pub mod codec; pub mod frontend; diff --git a/native/native-executors/tests/cluster.rs b/native/native-executors/tests/cluster.rs index 681cff1d9a2..5cd539a27cd 100644 --- a/native/native-executors/tests/cluster.rs +++ b/native/native-executors/tests/cluster.rs @@ -43,6 +43,8 @@ struct Cluster { processes: Vec, work_dir: PathBuf, flight_url: String, + /// The scheduler's log, with this crate's messages at info level. + scheduler_log: PathBuf, } /// Held while a cluster starts, so that two tests don't pick the same free port in between. @@ -67,6 +69,10 @@ impl Cluster { std::process::id() )), flight_url: format!("http://127.0.0.1:{flight_port}"), + scheduler_log: std::env::temp_dir().join(format!( + "comet-scheduler-{}-{flight_port}.log", + std::process::id() + )), }; let mut env = vec![ ("COMET_SCHEDULER_BIND_HOST", "127.0.0.1".to_string()), @@ -74,7 +80,13 @@ impl Cluster { ("COMET_SCHEDULER_FLIGHT_PORT", flight_port.to_string()), ]; env.extend_from_slice(scheduler_env); - cluster.spawn(env!("CARGO_BIN_EXE_comet-scheduler"), &env); + let log = std::fs::File::create(&cluster.scheduler_log).unwrap(); + cluster.spawn_logging( + env!("CARGO_BIN_EXE_comet-scheduler"), + &env, + "warn,comet_native_executors=info", + Stdio::from(log), + ); wait_for(scheduler_port); wait_for(flight_port); let work_dir = cluster.work_dir.to_string_lossy().into_owned(); @@ -95,10 +107,15 @@ impl Cluster { } fn spawn(&mut self, binary: &str, env: &[(&str, String)]) { + self.spawn_logging(binary, env, "warn", Stdio::inherit()); + } + + fn spawn_logging(&mut self, binary: &str, env: &[(&str, String)], log: &str, stderr: Stdio) { let child = Command::new(binary) .envs(env.iter().map(|(key, value)| (*key, value.as_str()))) - .env("RUST_LOG", "warn") + .env("RUST_LOG", log) .stdout(Stdio::null()) + .stderr(stderr) .spawn() .unwrap_or_else(|e| panic!("Cannot start {binary}: {e}")); self.processes.push(child); @@ -138,6 +155,7 @@ impl Drop for Cluster { let _ = process.wait(); } let _ = std::fs::remove_dir_all(&self.work_dir); + let _ = std::fs::remove_file(&self.scheduler_log); } } @@ -232,6 +250,62 @@ async fn sorts_across_a_range_exchange_by_merging_sorted_partitions() { assert_eq!(values, (0..1000).rev().collect::>()); } +#[tokio::test(flavor = "multi_thread")] +async fn broadcasts_a_build_side_that_measures_small_into_the_probe_side_stage() { + // Spark's estimates make the build side worth measuring before the probe side is shuffled: + // under Ballista's maximum estimate multiple, and the probe side many times larger. It + // measures far under the broadcast threshold, so the join runs in the probe side's stage. + let cluster = Cluster::start(); + let mut probe = range_stage(10_000, 4); + probe.estimated_output_bytes = Some(1 << 40); + let mut build = range_stage(100, 2); + build.estimated_output_bytes = Some(1 << 30); + let request = query(vec![probe, build, joining_stage(0, 1, 3)]); + + let partitions = cluster.run(&request).await; + + let rows = partitions.concat(); + assert_eq!( + sorted_values::(&rows), + (0..100).collect::>() + ); + for batch in &rows { + assert_eq!(batch.column(0), batch.column(1)); + } + let log = std::fs::read_to_string(&cluster.scheduler_log).unwrap(); + assert!(log.contains("Holding back a join's probe side"), "{log}"); + assert!( + log.contains("broadcasting it to the probe side's stage"), + "{log}" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn shuffles_the_probe_side_when_the_build_side_measures_large() { + // With a broadcast threshold of one byte, a held join's build side always measures too + // large, so the probe side's stage is released as it was and the join is shuffled. + let cluster = Cluster::start_with(&[( + "COMET_SCHEDULER_BALLISTA_CONFIG", + "ballista.optimizer.broadcast_join_threshold_bytes=1".to_string(), + )]); + let mut probe = range_stage(10_000, 4); + probe.estimated_output_bytes = Some(1 << 40); + let mut build = range_stage(100, 2); + build.estimated_output_bytes = Some(16); + let request = query(vec![probe, build, joining_stage(0, 1, 3)]); + + let partitions = cluster.run(&request).await; + + let rows = partitions.concat(); + assert_eq!( + sorted_values::(&rows), + (0..100).collect::>() + ); + let log = std::fs::read_to_string(&cluster.scheduler_log).unwrap(); + assert!(log.contains("Holding back a join's probe side"), "{log}"); + assert!(log.contains("shuffling the probe side"), "{log}"); +} + #[tokio::test(flavor = "multi_thread")] async fn refuses_a_query_from_another_build() { let cluster = Cluster::start(); diff --git a/native/native-executors/tests/common/mod.rs b/native/native-executors/tests/common/mod.rs index 60d56ec5c3d..b7e1b3acaa4 100644 --- a/native/native-executors/tests/common/mod.rs +++ b/native/native-executors/tests/common/mod.rs @@ -30,7 +30,8 @@ use datafusion_comet_proto::spark_expression::{ }; use datafusion_comet_proto::spark_native_executors::{QueryRequest, Stage, StageInput}; use datafusion_comet_proto::spark_operator::{ - operator::OpStruct, Operator, RangeScan, Scan, Sort, SparkStructField, + operator::OpStruct, BuildSide, HashJoin, JoinType, Operator, RangeScan, Scan, ShuffleScan, + Sort, SparkStructField, }; use datafusion_comet_proto::spark_partitioning::{ partitioning::PartitioningStruct, HashPartition, Partitioning, RangePartition, SinglePartition, @@ -218,3 +219,38 @@ where values.sort_unstable(); values } + +/// A stage that joins what stages `probe` and `build` send it through hash exchanges of +/// `num_partitions` partitions on their first column, as an inner hash join built on the build +/// side, and returns both sides' columns. +pub fn joining_stage(probe: u32, build: u32, num_partitions: i32) -> Stage { + let leaf = || Operator { + op_struct: Some(OpStruct::ShuffleScan(ShuffleScan { + fields: vec![long_type()], + source: "test".to_string(), + })), + ..Default::default() + }; + let input = |producer| StageInput { + producer, + partitioning: Some(hash_on_first_column(num_partitions)), + broadcast: false, + }; + Stage { + plan: Some(Operator { + children: vec![leaf(), leaf()], + op_struct: Some(OpStruct::HashJoin(HashJoin { + left_join_keys: vec![bound(0, long_type())], + right_join_keys: vec![bound(0, long_type())], + join_type: JoinType::Inner as i32, + build_side: BuildSide::BuildRight as i32, + ..Default::default() + })), + ..Default::default() + }), + inputs: vec![input(probe), input(build)], + num_partitions: num_partitions as u32, + output_schema: vec![field(long_type()), field(long_type())], + ..Default::default() + } +} diff --git a/native/proto/src/proto/native_executors.proto b/native/proto/src/proto/native_executors.proto index 9d084fe374b..8fdb2261be4 100644 --- a/native/proto/src/proto/native_executors.proto +++ b/native/proto/src/proto/native_executors.proto @@ -64,6 +64,9 @@ message Stage { // which needs bounds Spark would sample for, the cluster reads the producer's partitions as // they are, sorts each, and merges them in this order. repeated spark.spark_expression.Expr output_ordering = 6; + // Spark's estimate of the stage's output size in bytes, when it has one. The scheduler reads + // it to decide whether measuring a join's build side before shuffling its probe side pays off. + optional uint64 estimated_output_bytes = 7; } // How a scheduler sends a stage to its executors, inside the task plans Ballista ships. The @@ -76,6 +79,8 @@ message Fragment { // Whether every partition reads all of each input, one entry per input. repeated bool broadcast = 3; map conf = 4; + // Spark's estimate of the stage's output size in bytes, from Stage.estimated_output_bytes. + optional uint64 estimated_output_bytes = 5; } message StageInput { diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala index 15236b380fb..2f024b706d9 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala @@ -23,6 +23,7 @@ import java.util.IdentityHashMap import scala.collection.mutable.ArrayBuffer import scala.jdk.CollectionConverters._ +import scala.util.Try import org.apache.spark.{Partition, SparkContext, TaskContext} import org.apache.spark.rdd.RDD @@ -443,7 +444,7 @@ object CometNativeExecutors { // block's output to it as that writer does. val outputSchema = schema2Proto( plan.output.map(a => StructField(a.name, a.dataType, a.nullable, a.metadata))) - stages += Stage + val stage = Stage .newBuilder() .setPlan(blockPlan) .addAllInputs(inputs.asJava) @@ -451,12 +452,25 @@ object CometNativeExecutors { .setNumPartitions(numPartitions) .addAllOutputSchema(outputSchema.asJava) .addAllOutputOrdering(outputOrdering.asJava) - .build() + estimatedOutputBytes(plan).foreach(stage.setEstimatedOutputBytes) + stages += stage.build() val index = stages.length - 1 stageIndexes.put(plan, index) index } + /** + * Spark's estimate of the size of `plan`'s output, from the logical plan of the nearest node + * in it that has one. The scheduler uses it to decide whether a join's build side is worth + * measuring before its probe side is shuffled. + */ + private def estimatedOutputBytes(plan: SparkPlan): Option[Long] = + plan + .collectFirst { case node if node.logicalLink.isDefined => node.logicalLink.get } + .flatMap(logical => Try(logical.stats.sizeInBytes).toOption) + .filter(_ >= 0) + .map(_.min(Long.MaxValue).toLong) + /** * The exchanges that feed `block`, in the order its native plan's input leaves read them: * native shuffles and broadcasts, with a reused exchange resolved to the one it reuses. From 7e8fb40dec1602f5ae0b9c1f67c13d81950b3f73 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 2 Nov 2026 09:34:30 -0700 Subject: [PATCH 30/36] feat: measure a small build side before a pipelined query on Comet Native Executors A pipelined query places every task before any runs, so a join whose build side turns out small still shuffles its whole probe side. The scheduler now runs the build side of each movable join that Spark's estimates say is worth measuring as a pipelined query of its own first, and where it comes back under the broadcast threshold moves the join into the probe side's stage, reading the build side from memory. The rewrite is shared with the adaptive rule for blocking mode. Prototype for #7408. --- native/native-executors/src/adaptive.rs | 404 ++++++++++++++++------- native/native-executors/src/frontend.rs | 12 + native/native-executors/tests/cluster.rs | 79 ++++- 3 files changed, 364 insertions(+), 131 deletions(-) diff --git a/native/native-executors/src/adaptive.rs b/native/native-executors/src/adaptive.rs index b93b606664c..afb7c3607ad 100644 --- a/native/native-executors/src/adaptive.rs +++ b/native/native-executors/src/adaptive.rs @@ -32,6 +32,7 @@ //! build-side staging it uses Ballista's broadcast threshold, minimum probe ratio and maximum //! estimate multiple. +use std::future::Future; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; @@ -40,10 +41,13 @@ use ballista_scheduler::state::adaptive::{AdaptiveRule, AdaptiveRules, ExchangeE use comet::execution::native_executors::{ partition_descriptor, partition_plans, CometFragmentExec, }; -use datafusion::common::tree_node::{Transformed, TreeNode}; +use datafusion::arrow::array::RecordBatch; +use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; use datafusion::common::{internal_datafusion_err, Result}; use datafusion::config::ConfigOptions; +use datafusion::datasource::memory::MemorySourceConfig; use datafusion::physical_optimizer::PhysicalOptimizerRule; +use datafusion::physical_plan::repartition::RepartitionExec; use datafusion::physical_plan::statistics::StatisticsContext; use datafusion::physical_plan::{ ChildrenPropertiesMode, ExecutionPlan, Partitioning, ReplaceChildrenOptions, StatisticsArgs, @@ -169,6 +173,40 @@ fn hash_exchange(input: &Arc) -> Option<(&ExchangeExec, &Come Some((exchange, producer)) } +/// The stage that produces a stage input, when the input is a hash shuffle of it: an +/// [`ExchangeExec`] in adaptive planning, or the `RepartitionExec` the stage graph is built with. +fn hash_shuffled_stage(input: &Arc) -> Option<&CometFragmentExec> { + if let Some((_, producer)) = hash_exchange(input) { + return Some(producer); + } + let repartition = input.downcast_ref::()?; + if !matches!(repartition.partitioning(), Partitioning::Hash(..)) { + return None; + } + repartition.input().downcast_ref::() +} + +/// Spark's estimates of a join's probe and build sides, when they say measuring the build side +/// before shuffling the probe side could pay off: the build side within Ballista's maximum +/// estimate multiple of the broadcast threshold, and the probe side many times larger. +fn worth_measuring( + probe: &CometFragmentExec, + build: &CometFragmentExec, + config: &BallistaConfig, +) -> Option<(u64, u64)> { + let threshold = config.broadcast_join_threshold_bytes() as u64; + if threshold == 0 { + return None; + } + let max_build = + threshold.saturating_mul(config.stage_build_side_max_estimate_multiple() as u64); + let min_probe_ratio = config.stage_build_side_min_probe_ratio() as u64; + let probe_bytes = probe.fragment().estimated_output_bytes?; + let build_bytes = build.fragment().estimated_output_bytes?; + (build_bytes <= max_build && probe_bytes >= build_bytes.saturating_mul(min_probe_ratio)) + .then_some((probe_bytes, build_bytes)) +} + fn ballista_config(config: &ConfigOptions) -> BallistaConfig { config .extensions @@ -193,13 +231,9 @@ impl PhysicalOptimizerRule for HoldProbeStages { config: &ConfigOptions, ) -> Result> { let config = ballista_config(config); - let threshold = config.broadcast_join_threshold_bytes() as u64; - if threshold == 0 || !config.adaptive_join_enabled() { + if !config.adaptive_join_enabled() { return Ok(plan); } - let max_build = - threshold.saturating_mul(config.stage_build_side_max_estimate_multiple() as u64); - let min_probe_ratio = config.stage_build_side_min_probe_ratio() as u64; Ok(plan .transform_up(|node| { let Some(fragment) = node.downcast_ref::() else { @@ -221,17 +255,14 @@ impl PhysicalOptimizerRule for HoldProbeStages { ) else { continue; }; - let (Some(probe_bytes), Some(build_bytes)) = ( - probe_stage.fragment().estimated_output_bytes, - build_stage.fragment().estimated_output_bytes, - ) else { + let Some((probe_bytes, build_bytes)) = + worth_measuring(probe_stage, build_stage, &config) + else { continue; }; if probe.shuffle_created() || build.shuffle_created() || probe.is_inactive_stage() - || build_bytes > max_build - || probe_bytes < build_bytes.saturating_mul(min_probe_ratio) { continue; } @@ -351,7 +382,8 @@ impl BroadcastIntoProbe { } ); let decided = if small { - self.broadcast_into_probe(fragment, &block, &join)? + let broadcast = build.to_broadcast(self.plan_ids.fetch_add(1, Ordering::Relaxed)); + move_join_into_probe(fragment, &block, &join, Arc::new(broadcast))? } else { let mut children = fragment .children() @@ -365,120 +397,118 @@ impl BroadcastIntoProbe { } Ok(None) } +} - /// `consumer` with `join` moved into the stage that produces its probe side, and its build - /// side broadcast to that stage. - fn broadcast_into_probe( - &self, - consumer: &CometFragmentExec, - block: &Operator, - join: &MovableJoin, - ) -> Result> { - let inputs = &consumer.children()[1..]; - let (_, probe_stage) = hash_exchange(inputs[join.probe_input]) - .ok_or_else(|| internal_datafusion_err!("The join's probe side is not an exchange"))?; - let (build, build_stage) = hash_exchange(inputs[join.build_input]) - .ok_or_else(|| internal_datafusion_err!("The join's build side is not an exchange"))?; - let join_op = node_at(block, &join.path); - let build_leaf = &join_op.children[1]; - - // The probe side's stage runs the join on its output, reading the build side whole as - // its last input. The join becomes the root of the stage's block, so it takes over the - // block's pool of SQL texts. - let into_probe = |root: Operator| -> Operator { - let mut root = root; - let mut moved = join_op.clone(); - moved.sql_text_pool = std::mem::take(&mut root.sql_text_pool); - moved.children = vec![root, build_leaf.clone()]; - moved - }; - let probe_fragment = probe_stage.fragment(); - let block_of_probe = match block_of(probe_stage)? { - Some(root) => into_probe(root).encode_to_vec(), - None => vec![], - }; - let descriptor = Arc::clone(probe_stage.children()[0]); - let own_plans = partition_plans(&descriptor)?; - let descriptor = if own_plans.is_empty() { - descriptor - } else { - let plans = own_plans - .iter() - .map(|plan| { - let root = Operator::decode(plan.as_slice()).map_err(|e| { - internal_datafusion_err!("Cannot decode a partition's block: {e}") - })?; - Ok(into_probe(root).encode_to_vec()) - }) - .collect::>>()?; - partition_descriptor(plans.len(), &plans)? - }; - let mut broadcast = probe_fragment.broadcast.clone(); - broadcast.push(true); - let mut output_schema = probe_fragment.output_schema.clone(); - output_schema.extend(build_stage.fragment().output_schema.iter().cloned()); - let joined = Fragment { - block: block_of_probe, - output_schema, - broadcast, - conf: probe_fragment.conf.clone(), - estimated_output_bytes: None, - }; - let mut children = vec![descriptor]; - children.extend( - probe_stage.children()[1..] - .iter() - .map(|child| Arc::clone(*child)), - ); - children.push(Arc::new( - build.to_broadcast(self.plan_ids.fetch_add(1, Ordering::Relaxed)), - )); - let joined: Arc = - Arc::new(CometFragmentExec::try_new(Arc::new(joined), children)?); - let exchange = with_children(inputs[join.probe_input], vec![joined])?; - let exchange = exchange - .downcast_ref::() - .ok_or_else(|| internal_datafusion_err!("An exchange rebuilt as something else"))? - .with_inactive_stage(false); - - // The consumer reads the join's output where it read the join's probe side, and no - // longer reads the build side. - let mut fields = leaf_fields(&join_op.children[0]).to_vec(); - fields.extend(leaf_fields(build_leaf).iter().cloned()); - let leaf = Operator { - children: vec![], - plan_id: join_op.plan_id, - sql_text_pool: if join.path.is_empty() { - join_op.sql_text_pool.clone() - } else { - vec![] - }, - op_struct: Some(OpStruct::ShuffleScan(ShuffleScan { - fields, - source: "a join moved into the probe side's stage".to_string(), - })), - }; - let mut consumer_block = block.clone(); - replace_at(&mut consumer_block, &join.path, leaf); - let mut consumer_broadcast = consumer.fragment().broadcast.clone(); - consumer_broadcast.remove(join.build_input); - let consumer_fragment = Fragment { - block: consumer_block.encode_to_vec(), - broadcast: consumer_broadcast, - ..consumer.fragment().as_ref().clone() - }; - let mut children = consumer - .children() - .into_iter() - .map(Arc::clone) - .collect::>(); - children[1 + join.probe_input] = Arc::new(exchange); - children.remove(1 + join.build_input); - Ok(Arc::new(CometFragmentExec::try_new( - Arc::new(consumer_fragment), - children, - )?)) +/// `consumer` with `join` moved into the stage that produces its probe side, which reads the +/// build side as `broadcast`: one partition of the build side's whole output. +fn move_join_into_probe( + consumer: &CometFragmentExec, + block: &Operator, + join: &MovableJoin, + broadcast: Arc, +) -> Result> { + let inputs = &consumer.children()[1..]; + let probe_stage = hash_shuffled_stage(inputs[join.probe_input]) + .ok_or_else(|| internal_datafusion_err!("The join's probe side is not a shuffle"))?; + let build_stage = hash_shuffled_stage(inputs[join.build_input]) + .ok_or_else(|| internal_datafusion_err!("The join's build side is not a shuffle"))?; + let join_op = node_at(block, &join.path); + let build_leaf = &join_op.children[1]; + + // The probe side's stage runs the join on its output, reading the build side whole as + // its last input. The join becomes the root of the stage's block, so it takes over the + // block's pool of SQL texts. + let into_probe = |root: Operator| -> Operator { + let mut root = root; + let mut moved = join_op.clone(); + moved.sql_text_pool = std::mem::take(&mut root.sql_text_pool); + moved.children = vec![root, build_leaf.clone()]; + moved + }; + let probe_fragment = probe_stage.fragment(); + let block_of_probe = match block_of(probe_stage)? { + Some(root) => into_probe(root).encode_to_vec(), + None => vec![], + }; + let descriptor = Arc::clone(probe_stage.children()[0]); + let own_plans = partition_plans(&descriptor)?; + let descriptor = if own_plans.is_empty() { + descriptor + } else { + let plans = own_plans + .iter() + .map(|plan| { + let root = Operator::decode(plan.as_slice()).map_err(|e| { + internal_datafusion_err!("Cannot decode a partition's block: {e}") + })?; + Ok(into_probe(root).encode_to_vec()) + }) + .collect::>>()?; + partition_descriptor(plans.len(), &plans)? + }; + let mut broadcast_flags = probe_fragment.broadcast.clone(); + broadcast_flags.push(true); + let mut output_schema = probe_fragment.output_schema.clone(); + output_schema.extend(build_stage.fragment().output_schema.iter().cloned()); + let joined = Fragment { + block: block_of_probe, + output_schema, + broadcast: broadcast_flags, + conf: probe_fragment.conf.clone(), + estimated_output_bytes: None, + }; + let mut children = vec![descriptor]; + children.extend( + probe_stage.children()[1..] + .iter() + .map(|child| Arc::clone(*child)), + ); + children.push(broadcast); + let joined: Arc = + Arc::new(CometFragmentExec::try_new(Arc::new(joined), children)?); + // A held stage runs once its join is decided. + let mut exchange = with_children(inputs[join.probe_input], vec![joined])?; + if let Some(held) = exchange.downcast_ref::() { + exchange = Arc::new(held.with_inactive_stage(false)); } + + // The consumer reads the join's output where it read the join's probe side, and no + // longer reads the build side. + let mut fields = leaf_fields(&join_op.children[0]).to_vec(); + fields.extend(leaf_fields(build_leaf).iter().cloned()); + let leaf = Operator { + children: vec![], + plan_id: join_op.plan_id, + sql_text_pool: if join.path.is_empty() { + join_op.sql_text_pool.clone() + } else { + vec![] + }, + op_struct: Some(OpStruct::ShuffleScan(ShuffleScan { + fields, + source: "a join moved into the probe side's stage".to_string(), + })), + }; + let mut consumer_block = block.clone(); + replace_at(&mut consumer_block, &join.path, leaf); + let mut consumer_broadcast = consumer.fragment().broadcast.clone(); + consumer_broadcast.remove(join.build_input); + let consumer_fragment = Fragment { + block: consumer_block.encode_to_vec(), + broadcast: consumer_broadcast, + ..consumer.fragment().as_ref().clone() + }; + let mut children = consumer + .children() + .into_iter() + .map(Arc::clone) + .collect::>(); + children[1 + join.probe_input] = exchange; + children.remove(1 + join.build_input); + Ok(Arc::new(CometFragmentExec::try_new( + Arc::new(consumer_fragment), + children, + )?)) } fn replace_at(block: &mut Operator, path: &[usize], replacement: Operator) { @@ -497,3 +527,127 @@ fn with_children( ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), ) } + +/// Measures, before a pipelined query runs, the build side of each [`MovableJoin`] that +/// [`worth_measuring`] picks, and moves the join into its probe side's stage where the build side +/// comes back under the broadcast threshold. +/// +/// A pipelined query places every task before any runs, so it can't hold a stage back the way +/// [`HoldProbeStages`] does. Instead `run` runs the build side's stage as a query of its own and +/// returns its output, and the probe side's stage reads that output from memory as a broadcast. +/// A build side that comes back large runs again as part of the query. +pub async fn broadcast_measured_build_sides( + plan: Arc, + config: &BallistaConfig, + run: F, +) -> Result> +where + F: Fn(Arc) -> Fut, + Fut: Future>>, +{ + let threshold = config.broadcast_join_threshold_bytes(); + let mut plan = plan; + let mut measured: Vec> = vec![]; + while let Some(build) = next_build_side_to_measure(&plan, config, &measured)? { + measured.push(Arc::clone(&build)); + let batches = run(Arc::clone(&build)).await?; + let bytes: usize = batches.iter().map(RecordBatch::get_array_memory_size).sum(); + let rows: usize = batches.iter().map(RecordBatch::num_rows).sum(); + let small = bytes <= threshold; + info!( + "A join's build side measured {bytes} bytes and {rows} rows before the query: {}", + if small { + "broadcasting it to the probe side's stage" + } else { + "shuffling the probe side" + } + ); + if small { + let source = MemorySourceConfig::try_new_exec(&[batches], build.schema(), None)?; + plan = move_join_reading(plan, &build, source)?; + } + } + Ok(plan) +} + +/// The stage of the first build side in `plan` worth measuring that isn't in `measured`. +fn next_build_side_to_measure( + plan: &Arc, + config: &BallistaConfig, + measured: &[Arc], +) -> Result>> { + let mut found = None; + plan.apply(|node| { + let Some(fragment) = node.downcast_ref::() else { + return Ok(TreeNodeRecursion::Continue); + }; + let Some(block) = block_of(fragment)? else { + return Ok(TreeNodeRecursion::Continue); + }; + let inputs = &fragment.children()[1..]; + for join in movable_joins(&block) { + let (Some(probe_stage), Some(build_stage)) = ( + hash_shuffled_stage(inputs[join.probe_input]), + hash_shuffled_stage(inputs[join.build_input]), + ) else { + continue; + }; + let build = inputs[join.build_input].children()[0]; + if measured.iter().any(|seen| same_node(seen, build)) { + continue; + } + if let Some((probe_bytes, build_bytes)) = + worth_measuring(probe_stage, build_stage, config) + { + info!( + "Measuring a join's build side (estimated {build_bytes} bytes) before its \ + probe side (estimated {probe_bytes} bytes) is planned" + ); + found = Some(Arc::clone(build)); + return Ok(TreeNodeRecursion::Stop); + } + } + Ok(TreeNodeRecursion::Continue) + })?; + Ok(found) +} + +/// `plan` with the join whose build side is the stage `build` moved into its probe side's +/// stage, reading the build side from `source`. +fn move_join_reading( + plan: Arc, + build: &Arc, + source: Arc, +) -> Result> { + let mut moved = false; + Ok(plan + .transform_up(|node| { + if moved { + return Ok(Transformed::no(node)); + } + let Some(fragment) = node.downcast_ref::() else { + return Ok(Transformed::no(node)); + }; + let Some(block) = block_of(fragment)? else { + return Ok(Transformed::no(node)); + }; + let inputs = &fragment.children()[1..]; + for join in movable_joins(&block) { + if hash_shuffled_stage(inputs[join.probe_input]).is_none() + || hash_shuffled_stage(inputs[join.build_input]).is_none() + || !same_node(inputs[join.build_input].children()[0], build) + { + continue; + } + moved = true; + let rewritten = move_join_into_probe(fragment, &block, &join, Arc::clone(&source))?; + return Ok(Transformed::yes(rewritten)); + } + Ok(Transformed::no(node)) + })? + .data) +} + +fn same_node(a: &Arc, b: &Arc) -> bool { + std::ptr::addr_eq(Arc::as_ptr(a), Arc::as_ptr(b)) +} diff --git a/native/native-executors/src/frontend.rs b/native/native-executors/src/frontend.rs index f457cd04d27..b693a5a0fde 100644 --- a/native/native-executors/src/frontend.rs +++ b/native/native-executors/src/frontend.rs @@ -56,6 +56,7 @@ use prost::Message; use prost_types::Timestamp; use tonic::{Request, Response, Status, Streaming}; +use crate::adaptive::broadcast_measured_build_sides; use crate::assembly::{build_plan, check_version}; /// The Ballista session every query runs in. Every query uses Ballista's default settings, and @@ -377,6 +378,17 @@ impl FlightService for CometFlightService { .map_err(|e| Status::invalid_argument(e.to_string()))?; let mut info = FlightInfo::new(); if pipelined { + let config = self.session.ballista_config(); + let plan = broadcast_measured_build_sides(plan, &config, |build| async move { + let (_, result) = run_pipelined(&self.scheduler, &self.session, build) + .await + .map_err(|status| { + datafusion::error::DataFusionError::External(Box::new(status)) + })?; + Ok(result.partitions.concat()) + }) + .await + .map_err(|e| Status::internal(format!("Cannot measure a join's build side: {e}")))?; let (job_id, result) = run_pipelined(&self.scheduler, &self.session, plan).await?; let now = SystemTime::now() .duration_since(UNIX_EPOCH) diff --git a/native/native-executors/tests/cluster.rs b/native/native-executors/tests/cluster.rs index 5cd539a27cd..80c5cb0b504 100644 --- a/native/native-executors/tests/cluster.rs +++ b/native/native-executors/tests/cluster.rs @@ -134,15 +134,24 @@ impl Cluster { let info = self.get_flight_info(request).await.unwrap(); let mut partitions = vec![]; for endpoint in &info.endpoint { - partitions.push(read(endpoint).await.unwrap()); + partitions.push(read(endpoint, &self.flight_url).await.unwrap()); } partitions } } -/// Reads a result partition from the executor that holds it. -async fn read(endpoint: &FlightEndpoint) -> Result, FlightError> { - let mut executor = connect(&endpoint.location[0].uri).await; +/// Reads a result partition from the executor that holds it, or from the scheduler at +/// `scheduler_url` for an endpoint with no location, as the scheduler serves a pipelined query's +/// results itself. +async fn read( + endpoint: &FlightEndpoint, + scheduler_url: &str, +) -> Result, FlightError> { + let url = endpoint + .location + .first() + .map_or(scheduler_url, |location| location.uri.as_str()); + let mut executor = connect(url).await; let ticket = endpoint.ticket.clone().unwrap(); within_time_limit(async { executor.do_get(ticket).await?.try_collect().await }).await } @@ -306,6 +315,64 @@ async fn shuffles_the_probe_side_when_the_build_side_measures_large() { assert!(log.contains("shuffling the probe side"), "{log}"); } +#[tokio::test(flavor = "multi_thread")] +async fn measures_a_build_side_before_a_pipelined_query_and_broadcasts_it() { + // A pipelined query places every task before any runs, so the scheduler runs the build side + // first. It measures far under the broadcast threshold, so the join runs in the probe side's + // stage, reading the build side from memory. + let cluster = Cluster::start_with(&[( + "COMET_SCHEDULER_BALLISTA_CONFIG", + "ballista.execution.mode=pipelined".to_string(), + )]); + let mut probe = range_stage(10_000, 4); + probe.estimated_output_bytes = Some(1 << 40); + let mut build = range_stage(100, 2); + build.estimated_output_bytes = Some(1 << 30); + let request = query(vec![probe, build, joining_stage(0, 1, 3)]); + + let partitions = cluster.run(&request).await; + + let rows = partitions.concat(); + assert_eq!( + sorted_values::(&rows), + (0..100).collect::>() + ); + for batch in &rows { + assert_eq!(batch.column(0), batch.column(1)); + } + let log = std::fs::read_to_string(&cluster.scheduler_log).unwrap(); + assert!( + log.contains("before the query: broadcasting it to the probe side's stage"), + "{log}" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn runs_a_pipelined_query_as_planned_when_the_build_side_measures_large() { + let cluster = Cluster::start_with(&[( + "COMET_SCHEDULER_BALLISTA_CONFIG", + "ballista.execution.mode=pipelined,ballista.optimizer.broadcast_join_threshold_bytes=1" + .to_string(), + )]); + let mut probe = range_stage(10_000, 4); + probe.estimated_output_bytes = Some(1 << 40); + let mut build = range_stage(100, 2); + build.estimated_output_bytes = Some(16); + let request = query(vec![probe, build, joining_stage(0, 1, 3)]); + + let partitions = cluster.run(&request).await; + + assert_eq!( + sorted_values::(&partitions.concat()), + (0..100).collect::>() + ); + let log = std::fs::read_to_string(&cluster.scheduler_log).unwrap(); + assert!( + log.contains("before the query: shuffling the probe side"), + "{log}" + ); +} + #[tokio::test(flavor = "multi_thread")] async fn refuses_a_query_from_another_build() { let cluster = Cluster::start(); @@ -346,7 +413,7 @@ async fn endpoints_say_when_their_results_are_deleted() { assert!(expires_at <= SystemTime::now() + retention); // A client that reads one partition and then takes too long finds the other gone. - read(&info.endpoint[0]).await.unwrap(); + read(&info.endpoint[0], &cluster.flight_url).await.unwrap(); tokio::time::sleep( expires_at .duration_since(SystemTime::now()) @@ -354,7 +421,7 @@ async fn endpoints_say_when_their_results_are_deleted() { ) .await; let deadline = Instant::now() + Duration::from_secs(60); - while read(&info.endpoint[1]).await.is_ok() { + while read(&info.endpoint[1], &cluster.flight_url).await.is_ok() { assert!( Instant::now() < deadline, "A result partition was still readable a minute after its endpoint expired" From 5000f3a837ca1d24ff9cd8233ad4c5d2bea4aa50 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 2 Nov 2026 10:38:53 -0700 Subject: [PATCH 31/36] feat: convert input rows to partial states in Comet's double average A partial aggregate that stops aggregating sends each row on as a partial state, which DataFusion builds with GroupsAccumulator::convert_to_state. Comet's Float64 AvgGroupsAccumulator, which runs Spark's average of a decimal as avg(UnscaledValue(x)), returned NotImplemented. Each row's state is now its value and a count of one, or a zero sum and count for a null or filtered-out row, what state() emits for such a group. --- native/spark-expr/src/agg_funcs/avg.rs | 55 ++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/native/spark-expr/src/agg_funcs/avg.rs b/native/spark-expr/src/agg_funcs/avg.rs index 2b2422ce775..c44d30eda2a 100644 --- a/native/spark-expr/src/agg_funcs/avg.rs +++ b/native/spark-expr/src/agg_funcs/avg.rs @@ -334,12 +334,33 @@ where ]) } + /// Each row's own partial state, as `state` emits it: the value and a count of one, or a + /// zero sum and count for a null or filtered-out row. fn convert_to_state( &self, - _values: &[ArrayRef], - _opt_filter: Option<&arrow::array::BooleanArray>, + values: &[ArrayRef], + opt_filter: Option<&arrow::array::BooleanArray>, ) -> Result> { - not_impl_err!("Input batch conversion to state not implemented") + let values = values[0].as_primitive::(); + let mut sums = Vec::with_capacity(values.len()); + let mut counts = Vec::with_capacity(values.len()); + for idx in 0..values.len() { + let included = values.is_valid(idx) + && opt_filter.is_none_or(|filter| filter.is_valid(idx) && filter.value(idx)); + if included { + sums.push(values.value(idx)); + counts.push(1); + } else { + sums.push(T::default_value()); + counts.push(0); + } + } + let sums = PrimitiveArray::::new(sums.into(), None) + .with_data_type(self.return_data_type.clone()); + Ok(vec![ + Arc::new(sums) as ArrayRef, + Arc::new(Int64Array::new(counts.into(), None)) as ArrayRef, + ]) } fn size(&self) -> usize { @@ -365,4 +386,32 @@ mod tests { assert_eq!(partial.evaluate()?, ScalarValue::Float64(None)); Ok(()) } + + /// Converting rows straight to partial states, as a partial aggregate that stops aggregating + /// does, gives each row the state that aggregating it alone into a group would. + #[test] + fn convert_to_state_matches_one_row_groups() -> Result<()> { + use arrow::array::BooleanArray; + let values: ArrayRef = Arc::new(Float64Array::from(vec![ + Some(1.5), + None, + Some(-2.0), + Some(4.0), + ])); + let filter = BooleanArray::from(vec![Some(true), Some(true), Some(true), Some(false)]); + let new_accumulator = || { + AvgGroupsAccumulator::::new( + &Float64, + |sum: f64, count: i64| Ok(sum / count as f64), + ) + }; + + let converted = + new_accumulator().convert_to_state(&[Arc::clone(&values)], Some(&filter))?; + + let mut grouped = new_accumulator(); + grouped.update_batch(&[values], &[0, 1, 2, 3], Some(&filter), 4)?; + assert_eq!(converted, grouped.state(EmitTo::All)?); + Ok(()) + } } From 0a4495a9a62dfd68ffbe4fad8e5219ec36545865 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 2 Nov 2026 10:38:54 -0700 Subject: [PATCH 32/36] perf: let a partial aggregate on Comet Native Executors stop aggregating, including AVG configure_skip_partial_aggregation only admitted a plan whose root is a ShuffleWriter. A Native Executors fragment has none, since Ballista writes the shuffle, so a fragment never skipped even with spark.comet.exec.aggregate.skipPartial.enabled. A fragment's session now marks it, and the check treats it as feeding a shuffle. Single-argument AVG is admitted next to COUNT, which applies to Comet on Spark too. On TPC-H q17 at SF1000 in pipelined mode this cut the query from about 277 s to 242 s, and stopped an executor from being OOM-killed (#7406). --- native/core/src/execution/jni_api.rs | 40 ++++++++++++++++++- .../execution/native_executors/fragment.rs | 8 +++- .../scala/org/apache/comet/CometConf.scala | 2 +- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index 33c594d9ca8..de5e539625d 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -1034,6 +1034,8 @@ fn configure_skip_partial_aggregation(config: &mut SessionConfig, plan: &Operato && agg.agg_exprs.iter().all(|expr| { matches!(&expr.expr_struct, Some(AggExprStruct::Count(count)) if count.children.len() == 1) + || matches!(&expr.expr_struct, Some(AggExprStruct::Avg(avg)) + if avg.child.is_some()) }) } // PartialMerge consumes states, not rows. @@ -1057,6 +1059,9 @@ fn configure_skip_partial_aggregation(config: &mut SessionConfig, plan: &Operato } } +/// Set in the configuration of a Comet Native Executors fragment's session. +pub(crate) const NATIVE_EXECUTORS_FRAGMENT: &str = "spark.comet.nativeExecutors.fragment"; + /// Configure DataFusion session context. fn prepare_datafusion_session_context( batch_size: usize, @@ -1151,9 +1156,25 @@ pub(crate) fn comet_session_context( crate::execution::c_udf::policy::NativeUdfPolicy::from_spark_config(spark_config), )); + // A Comet Native Executors fragment has no ShuffleWriter root, but its output goes to the + // cluster's shuffle all the same, so it is checked as if it had one. + let fragment_root; + let gate_plan = if spark_config + .get(NATIVE_EXECUTORS_FRAGMENT) + .is_some_and(|value| value == "true") + { + fragment_root = Operator { + op_struct: Some(OpStruct::ShuffleWriter(Default::default())), + children: vec![spark_plan.clone()], + ..Default::default() + }; + &fragment_root + } else { + spark_plan + }; configure_skip_partial_aggregation( &mut session_config, - spark_plan, + gate_plan, spark_config.get_bool(COMET_EXEC_AGGREGATE_SKIP_PARTIAL_ENABLED), ); @@ -2712,7 +2733,7 @@ mod tests { use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::ExecutionPlan; use datafusion_comet_proto::spark_expression; - use datafusion_comet_proto::spark_expression::{AggExpr, Count, Expr, Sum}; + use datafusion_comet_proto::spark_expression::{AggExpr, Avg, Count, Expr, Sum}; use datafusion_comet_proto::spark_operator::{HashAggregate, ShuffleWriter}; use std::cell::Cell; use std::future::Future; @@ -2729,6 +2750,13 @@ mod tests { expr_struct: Some(AggExprStruct::Sum(Sum::default())), ..Default::default() }; + let avg = |child: Option| AggExpr { + expr_struct: Some(AggExprStruct::Avg(Avg { + child, + ..Default::default() + })), + ..Default::default() + }; let partial = HashAggregate { grouping_exprs: vec![Expr::default()], agg_exprs: vec![count.clone()], @@ -2773,6 +2801,10 @@ mod tests { agg_exprs: vec![], ..partial.clone() }, + HashAggregate { + agg_exprs: vec![count.clone(), avg(Some(Expr::default()))], + ..partial.clone() + }, HashAggregate { agg_exprs: vec![count.clone(), count], ..partial.clone() @@ -2797,6 +2829,10 @@ mod tests { agg_exprs: vec![AggExpr::default()], ..partial.clone() }, + HashAggregate { + agg_exprs: vec![avg(None)], + ..partial.clone() + }, HashAggregate { agg_exprs: vec![AggExpr { expr_struct: Some(AggExprStruct::Count(Count { diff --git a/native/core/src/execution/native_executors/fragment.rs b/native/core/src/execution/native_executors/fragment.rs index 7814ae2c3d8..fdd67ab9280 100644 --- a/native/core/src/execution/native_executors/fragment.rs +++ b/native/core/src/execution/native_executors/fragment.rs @@ -42,7 +42,9 @@ use datafusion_comet_proto::spark_operator::{Operator, SparkStructField}; use futures::stream::BoxStream; use futures::{StreamExt, TryStreamExt}; -use crate::execution::jni_api::{comet_runtime_env, comet_session_context}; +use crate::execution::jni_api::{ + comet_runtime_env, comet_session_context, NATIVE_EXECUTORS_FRAGMENT, +}; use crate::execution::planner::{ align_shuffle_writer_input, convert_spark_types_to_arrow_schema, PhysicalPlanner, }; @@ -390,11 +392,13 @@ fn fragment_session( ) -> Result> { let mut runtime = comet_runtime_env(memory_pool); runtime.disk_manager = disk_manager; + let mut conf = conf.clone(); + conf.insert(NATIVE_EXECUTORS_FRAGMENT.to_string(), "true".to_string()); let session = comet_session_context( runtime, conf.get_usize(COMET_BATCH_SIZE, DEFAULT_BATCH_SIZE), 1, - conf, + &conf, spark_plan, ) .map_err(|e| DataFusionError::External(Box::new(e)))?; diff --git a/spark/src/main/scala/org/apache/comet/CometConf.scala b/spark/src/main/scala/org/apache/comet/CometConf.scala index 2d7ea89fa98..192e611c022 100644 --- a/spark/src/main/scala/org/apache/comet/CometConf.scala +++ b/spark/src/main/scala/org/apache/comet/CometConf.scala @@ -732,7 +732,7 @@ object CometConf extends ShimCometConf { "Experimental opt-in: let a native partial aggregate stop aggregating once its input " + "looks mostly distinct, and send the rest of the task's rows to the shuffle " + "unaggregated. Only applies to partial aggregates that feed Comet native shuffle " + - "and whose aggregate functions, if any, are all single-argument COUNT. The check " + + "and whose aggregate functions, if any, are all single-argument COUNT or AVG. The check " + "starts after the first 100,000 input rows of a task, and once aggregation stops " + "it does not resume, so a task whose keys repeat after a mostly distinct start can " + s"shuffle many times more rows than it would with this disabled. $TUNING_GUIDE.") From 500fa31602799d19ffcca043cc5bfd9ebe798a94 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 2 Nov 2026 10:38:39 -0700 Subject: [PATCH 33/36] build: build Comet Native Executors on Ballista's comet-native branch with its adaptive rules hook comet-native now carries apache/datafusion-ballista#2743, which the adaptive join rule needs, so the workspace builds against it again rather than a fork. The README describes how the scheduler handles a join whose build side turns out small. --- native/native-executors/Cargo.lock | 12 ++++++------ native/native-executors/Cargo.toml | 11 ++++------- native/native-executors/README.md | 24 ++++++++++++++++++++++-- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/native/native-executors/Cargo.lock b/native/native-executors/Cargo.lock index cb87cfa8597..f64f8737ef6 100644 --- a/native/native-executors/Cargo.lock +++ b/native/native-executors/Cargo.lock @@ -999,7 +999,7 @@ dependencies = [ [[package]] name = "ballista-core" version = "55.0.0" -source = "git+https://github.com/andygrove/datafusion-ballista?branch=comet-adaptive-rules#56e2d8a2c6c387f7cbd856950b7814777aa31f51" +source = "git+https://github.com/apache/datafusion-ballista?branch=comet-native#0f092ee494d47d611f6e83f3660e60a4e4038d8c" dependencies = [ "arrow-flight", "async-trait", @@ -1028,7 +1028,7 @@ dependencies = [ [[package]] name = "ballista-executor" version = "55.0.0" -source = "git+https://github.com/andygrove/datafusion-ballista?branch=comet-adaptive-rules#56e2d8a2c6c387f7cbd856950b7814777aa31f51" +source = "git+https://github.com/apache/datafusion-ballista?branch=comet-native#0f092ee494d47d611f6e83f3660e60a4e4038d8c" dependencies = [ "arrow", "arrow-flight", @@ -1057,7 +1057,7 @@ dependencies = [ [[package]] name = "ballista-scheduler" version = "55.0.0" -source = "git+https://github.com/andygrove/datafusion-ballista?branch=comet-adaptive-rules#56e2d8a2c6c387f7cbd856950b7814777aa31f51" +source = "git+https://github.com/apache/datafusion-ballista?branch=comet-native#0f092ee494d47d611f6e83f3660e60a4e4038d8c" dependencies = [ "arrow-flight", "async-trait", @@ -5073,8 +5073,8 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ - "heck 0.5.0", - "itertools 0.14.0", + "heck 0.4.1", + "itertools 0.13.0", "log", "multimap", "petgraph", @@ -5095,7 +5095,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.119", diff --git a/native/native-executors/Cargo.toml b/native/native-executors/Cargo.toml index 0d77383bc46..afe9739162b 100644 --- a/native/native-executors/Cargo.toml +++ b/native/native-executors/Cargo.toml @@ -21,11 +21,8 @@ # This is its own workspace, with its own lockfile, so that the main Comet build never depends # on Ballista. It builds Comet's native crates by path together with Ballista's `comet-native` # branch, which is Ballista's main branch with the enhancements Comet Native Executors need that -# haven't merged yet (apache/datafusion-ballista#2682, #2721, #2722 and #2723), on the same +# haven't merged yet (apache/datafusion-ballista#2682, #2721, #2722, #2723 and #2743), on the same # DataFusion major version. -# -# Prototype for apache/datafusion-comet#7391: builds against `comet-adaptive-rules` on a fork, -# which is `comet-native` plus a hook for adaptive planning rules of Comet's own. [workspace] [package] @@ -40,9 +37,9 @@ publish = false [dependencies] arrow-flight = "59.2.0" -ballista-core = { git = "https://github.com/andygrove/datafusion-ballista", branch = "comet-adaptive-rules" } -ballista-executor = { git = "https://github.com/andygrove/datafusion-ballista", branch = "comet-adaptive-rules", default-features = false, features = ["arrow-ipc-optimizations"] } -ballista-scheduler = { git = "https://github.com/andygrove/datafusion-ballista", branch = "comet-adaptive-rules", default-features = false } +ballista-core = { git = "https://github.com/apache/datafusion-ballista", branch = "comet-native" } +ballista-executor = { git = "https://github.com/apache/datafusion-ballista", branch = "comet-native", default-features = false, features = ["arrow-ipc-optimizations"] } +ballista-scheduler = { git = "https://github.com/apache/datafusion-ballista", branch = "comet-native", default-features = false } datafusion = { version = "55.2.0", default-features = false } # Comet's core crate, under the name of its library. Without the default hdfs-opendal feature, # which links libjvm through libhdfs. diff --git a/native/native-executors/README.md b/native/native-executors/README.md index 62eab4ab2cf..11d6acc0edd 100644 --- a/native/native-executors/README.md +++ b/native/native-executors/README.md @@ -74,6 +74,25 @@ A stage that fails to plan is refused, not run wrongly or loaded with code it na - A stage that reads files can't also read a hash exchange. Spark places the rows of a bucketed scan with Murmur3, so a join with a side shuffled by DataFusion's hash would return wrong rows. +## Joins whose build side turns out small + +Spark plans a join's sides from its estimates, and its estimate of a filtered relation ignores the +filter, so it shuffles both sides of a join whose build side is far smaller than it thinks. The +scheduler moves such a join into the stage that produces its probe side, with the build side +broadcast to it, so the probe side's exchange carries the join's output rather than all of its +input (`src/adaptive.rs`). It picks the joins Spark's estimates say are worth measuring: an inner +hash join built on the right, with column keys and no condition, whose build side Spark estimates +within Ballista's maximum estimate multiple of its broadcast threshold and whose probe side many +times larger. + +- In blocking mode, a rule in Ballista's adaptive planner holds the probe side's stage back until + the build side's stage finishes, then moves the join if the build side measured under + `ballista.optimizer.broadcast_join_threshold_bytes`, or releases the probe side's stage as it + was. +- In pipelined mode every task is placed before any runs, so the scheduler first runs the build + side as a query of its own, and the probe side's stage reads it from memory if it came back under + the threshold. + ## Building and running ```shell @@ -102,8 +121,9 @@ This workspace builds Comet's native crates by path against Ballista's main branch with the enhancements Comet Native Executors need that haven't merged yet ([#2682](https://github.com/apache/datafusion-ballista/pull/2682), [#2721](https://github.com/apache/datafusion-ballista/pull/2721), -[#2722](https://github.com/apache/datafusion-ballista/pull/2722) and -[#2723](https://github.com/apache/datafusion-ballista/pull/2723)), on the same DataFusion major +[#2722](https://github.com/apache/datafusion-ballista/pull/2722), +[#2723](https://github.com/apache/datafusion-ballista/pull/2723) and +[#2743](https://github.com/apache/datafusion-ballista/pull/2743)), on the same DataFusion major version. `Cargo.lock` pins the branch's commit. CI builds and tests the workspace in the `linux-test-native-executors` job. When Comet moves to a DataFusion major that Ballista hasn't caught up with, the upgrade can turn that job off until it does. Dependabot updates this From fc11da37ca999e74bf10d0f0b49262ba37f7bc1b Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 2 Nov 2026 10:58:05 -0700 Subject: [PATCH 34/36] style: wrap the skip-partial setting's description at 100 columns --- spark/src/main/scala/org/apache/comet/CometConf.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/CometConf.scala b/spark/src/main/scala/org/apache/comet/CometConf.scala index 192e611c022..bc042152c53 100644 --- a/spark/src/main/scala/org/apache/comet/CometConf.scala +++ b/spark/src/main/scala/org/apache/comet/CometConf.scala @@ -732,8 +732,8 @@ object CometConf extends ShimCometConf { "Experimental opt-in: let a native partial aggregate stop aggregating once its input " + "looks mostly distinct, and send the rest of the task's rows to the shuffle " + "unaggregated. Only applies to partial aggregates that feed Comet native shuffle " + - "and whose aggregate functions, if any, are all single-argument COUNT or AVG. The check " + - "starts after the first 100,000 input rows of a task, and once aggregation stops " + + "and whose aggregate functions, if any, are all single-argument COUNT or AVG. The " + + "check starts after the first 100,000 input rows of a task, and once aggregation stops " + "it does not resume, so a task whose keys repeat after a mostly distinct start can " + s"shuffle many times more rows than it would with this disabled. $TUNING_GUIDE.") .booleanConf From fc7179509229fbd5bf73e1d057d61cee077b7260 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 2 Nov 2026 12:12:01 -0700 Subject: [PATCH 35/36] fix: compile Ballista's event-log writer into comet-scheduler COMET_SCHEDULER_EVENT_LOG_DIR set the scheduler config's event_log_dir, but Ballista's EventLogWriter is built only with the scheduler's rest-api feature, which default-features = false had left out, so no event log was ever written. Enable rest-api (adds ballista-api-types, ballista-history and utoipa to the lockfile). --- native/native-executors/Cargo.lock | 55 ++++++++++++++++++++++++++++-- native/native-executors/Cargo.toml | 3 +- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/native/native-executors/Cargo.lock b/native/native-executors/Cargo.lock index f64f8737ef6..6fe21bb60da 100644 --- a/native/native-executors/Cargo.lock +++ b/native/native-executors/Cargo.lock @@ -996,6 +996,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "ballista-api-types" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion-ballista?branch=comet-native#0f092ee494d47d611f6e83f3660e60a4e4038d8c" +dependencies = [ + "serde", + "utoipa", +] + [[package]] name = "ballista-core" version = "55.0.0" @@ -1022,6 +1031,7 @@ dependencies = [ "tonic-prost", "tonic-prost-build", "url", + "utoipa", "uuid", ] @@ -1054,6 +1064,18 @@ dependencies = [ "uuid", ] +[[package]] +name = "ballista-history" +version = "55.0.0" +source = "git+https://github.com/apache/datafusion-ballista?branch=comet-native#0f092ee494d47d611f6e83f3660e60a4e4038d8c" +dependencies = [ + "ballista-api-types", + "log", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "ballista-scheduler" version = "55.0.0" @@ -1062,7 +1084,9 @@ dependencies = [ "arrow-flight", "async-trait", "axum", + "ballista-api-types", "ballista-core", + "ballista-history", "dashmap", "datafusion", "datafusion-proto", @@ -1076,11 +1100,13 @@ dependencies = [ "prost", "rand 0.10.3", "serde", + "serde_json", "tokio", "tokio-stream", "tonic", "tower-http 0.7.1", "url-escape", + "utoipa", "uuid", ] @@ -5073,8 +5099,8 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ - "heck 0.4.1", - "itertools 0.13.0", + "heck 0.5.0", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -5095,7 +5121,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.119", @@ -6877,6 +6903,29 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "utoipa" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8765fe27aeff71012a3f90fa474cf1df863a7d0dd81ed25de4e63fff2544994f" +dependencies = [ + "indexmap 2.14.2", + "serde", + "serde_json", + "utoipa-gen", +] + +[[package]] +name = "utoipa-gen" +version = "6.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d935f1c83fdf8b88f09bbe050d0cea78ea4afee6d7b0317403228c1200b2c8ab" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "uuid" version = "1.29.0" diff --git a/native/native-executors/Cargo.toml b/native/native-executors/Cargo.toml index afe9739162b..dead316f884 100644 --- a/native/native-executors/Cargo.toml +++ b/native/native-executors/Cargo.toml @@ -39,7 +39,8 @@ publish = false arrow-flight = "59.2.0" ballista-core = { git = "https://github.com/apache/datafusion-ballista", branch = "comet-native" } ballista-executor = { git = "https://github.com/apache/datafusion-ballista", branch = "comet-native", default-features = false, features = ["arrow-ipc-optimizations"] } -ballista-scheduler = { git = "https://github.com/apache/datafusion-ballista", branch = "comet-native", default-features = false } +# `rest-api` compiles the scheduler's event-log writer (COMET_SCHEDULER_EVENT_LOG_DIR) and its REST API. +ballista-scheduler = { git = "https://github.com/apache/datafusion-ballista", branch = "comet-native", default-features = false, features = ["rest-api"] } datafusion = { version = "55.2.0", default-features = false } # Comet's core crate, under the name of its library. Without the default hdfs-opendal feature, # which links libjvm through libhdfs. From eb0534c1f3b880b666e4326e233e964f9b79e3aa Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 2 Nov 2026 14:52:50 -0700 Subject: [PATCH 36/36] fix: link tracing for Ballista's rest-api feature Ballista's rest-api code calls tracing, which it links only for its binaries (build-binary), so ballista-scheduler with default-features = false and rest-api did not compile. --- native/native-executors/Cargo.lock | 7 ++++--- native/native-executors/Cargo.toml | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/native/native-executors/Cargo.lock b/native/native-executors/Cargo.lock index 6fe21bb60da..b70bba3af43 100644 --- a/native/native-executors/Cargo.lock +++ b/native/native-executors/Cargo.lock @@ -1105,6 +1105,7 @@ dependencies = [ "tokio-stream", "tonic", "tower-http 0.7.1", + "tracing", "url-escape", "utoipa", "uuid", @@ -5099,8 +5100,8 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ - "heck 0.5.0", - "itertools 0.14.0", + "heck 0.4.1", + "itertools 0.13.0", "log", "multimap", "petgraph", @@ -5121,7 +5122,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.119", diff --git a/native/native-executors/Cargo.toml b/native/native-executors/Cargo.toml index dead316f884..c298f5c9f08 100644 --- a/native/native-executors/Cargo.toml +++ b/native/native-executors/Cargo.toml @@ -40,7 +40,8 @@ arrow-flight = "59.2.0" ballista-core = { git = "https://github.com/apache/datafusion-ballista", branch = "comet-native" } ballista-executor = { git = "https://github.com/apache/datafusion-ballista", branch = "comet-native", default-features = false, features = ["arrow-ipc-optimizations"] } # `rest-api` compiles the scheduler's event-log writer (COMET_SCHEDULER_EVENT_LOG_DIR) and its REST API. -ballista-scheduler = { git = "https://github.com/apache/datafusion-ballista", branch = "comet-native", default-features = false, features = ["rest-api"] } +# Its code calls `tracing`, which Ballista links only for its binaries (`build-binary`), so it is added here. +ballista-scheduler = { git = "https://github.com/apache/datafusion-ballista", branch = "comet-native", default-features = false, features = ["rest-api", "tracing"] } datafusion = { version = "55.2.0", default-features = false } # Comet's core crate, under the name of its library. Without the default hdfs-opendal feature, # which links libjvm through libhdfs.