Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions crates/integration_tests/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,4 @@ tar = "0.4"
tempfile = "3"
serde_json = "1"
axum = { version = "0.7", features = ["macros", "tokio", "http1", "http2"] }
uuid = { version = "1", features = ["v4"] }
17 changes: 17 additions & 0 deletions crates/integrations/datafusion/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -149,4 +149,21 @@ let view = ctx.sql("SELECT * FROM analytics_view").await?;
let function = ctx.sql("SELECT plus_one(score) FROM scores").await?;
```

## Persistent CREATE TABLE AS SELECT

Use `CREATE TABLE AS SELECT` to materialize a query as a native Paimon table.
The query determines the destination schema. In this example, `scores` resolves
in the session's current catalog and database.

```sql
CREATE TABLE paimon.reporting.results
WITH ('bucket' = '-1')
AS SELECT id, score FROM scores WHERE score > 10;
```

`IF NOT EXISTS` skips an existing target. If population fails after creation,
the destination remains in the catalog. For supported types, concurrent DDL,
cancellation and recovery after a commit error, see the
[SQL reference](../../../docs/src/sql.md#create-table-as-select).

See the [documentation](https://paimon.apache.org/docs/rust/datafusion/) for getting started guide and more details.
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ pub(crate) async fn execute_analyze(
"ANALYZE TABLE requires COMPUTE STATISTICS".to_string(),
));
}
SQLContext::ensure_partition_command_target(table_name, "ANALYZE TABLE")?;
SQLContext::ensure_main_branch_base_table(table_name, "ANALYZE TABLE")?;
ctx.reject_temporary_write_target(table_name, "ANALYZE TABLE")
.await?;
let (catalog, _catalog_name, identifier) = ctx.resolve_catalog_and_table(table_name)?;
Expand Down
2 changes: 1 addition & 1 deletion crates/integrations/datafusion/src/format_partition_ddl.rs
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ pub(crate) async fn execute_show_partitions(
show_partitions: &ShowPartitionsStatement,
enable_ident_normalization: bool,
) -> DFResult<DataFrame> {
SQLContext::ensure_partition_command_target(&show_partitions.table_name, "SHOW PARTITIONS")?;
SQLContext::ensure_main_branch_base_table(&show_partitions.table_name, "SHOW PARTITIONS")?;
let (catalog, _catalog_name, identifier) =
ctx.resolve_catalog_and_table(&show_partitions.table_name)?;
let table = catalog
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ pub(crate) async fn execute_msck(ctx: &SQLContext, msck: &Msck) -> DFResult<Data
"MSCK requires the REPAIR keyword".to_string(),
));
}
SQLContext::ensure_partition_command_target(&msck.table_name, "MSCK REPAIR TABLE")?;
SQLContext::ensure_main_branch_base_table(&msck.table_name, "MSCK REPAIR TABLE")?;
ctx.reject_temporary_write_target(&msck.table_name, "MSCK REPAIR TABLE")
.await?;
let (catalog, _catalog_name, identifier) = ctx.resolve_catalog_and_table(&msck.table_name)?;
Expand Down
91 changes: 91 additions & 0 deletions crates/integrations/datafusion/src/physical_plan/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,98 @@ pub(crate) mod scan;
mod search_score;
pub(crate) mod sink;

use std::sync::Arc;

use datafusion::arrow::array::{make_array, ArrayData, ArrayRef};
use datafusion::arrow::compute::{can_cast_types, cast};
use datafusion::arrow::datatypes::{DataType, FieldRef, SchemaRef};
use datafusion::arrow::error::ArrowError;
use datafusion::arrow::record_batch::{RecordBatch, RecordBatchOptions};
use datafusion::error::{DataFusionError, Result as DFResult};

pub(crate) use audit_log::PaimonAuditLogScan;
pub use scan::PaimonTableScan;
pub(crate) use search_score::{SearchScoreExec, SearchScoreOutputColumn};
pub use sink::PaimonDataSink;

// Arrow casts maps only when their ordering flags agree. Removing the sorted
// guarantee preserves the buffers, including offsets and nulls in nested maps.
fn normalize_map_ordering(data: ArrayData) -> Result<ArrayData, ArrowError> {
use DataType::*;
if data.child_data().is_empty() {
return Ok(data);
}
let children = data
.child_data()
.iter()
.cloned()
.map(normalize_map_ordering)
.collect::<Result<Vec<_>, _>>()?;
let field = |original: &FieldRef, index: usize| {
Arc::new(
original
.as_ref()
.clone()
.with_data_type(children[index].data_type().clone()),
)
};
let data_type = match data.data_type() {
Map(entries, _) => Map(field(entries, 0), false),
Struct(fields) => Struct(
fields
.iter()
.enumerate()
.map(|(i, f)| field(f, i))
.collect(),
),
List(f) => List(field(f, 0)),
LargeList(f) => LargeList(field(f, 0)),
FixedSizeList(f, size) => FixedSizeList(field(f, 0), *size),
Dictionary(key, _) => Dictionary(key.clone(), Box::new(children[0].data_type().clone())),
_ => return Ok(data),
};
if &data_type == data.data_type() {
return Ok(data);
}
data.into_builder()
.data_type(data_type)
.child_data(children)
.build()
}

fn cast_column(column: &ArrayRef, target: &DataType) -> Result<ArrayRef, ArrowError> {
if column.data_type() == target {
return Ok(Arc::clone(column));
}
if !can_cast_types(column.data_type(), target) {
let data = normalize_map_ordering(column.to_data())?;
if data.data_type() != column.data_type() {
return cast(&make_array(data), target);
}
}
cast(column, target)
}

pub(crate) fn cast_batch_to_schema(
batch: RecordBatch,
schema: &SchemaRef,
) -> DFResult<RecordBatch> {
if batch.schema_ref() == schema {
return Ok(batch);
}
if batch.num_columns() != schema.fields().len() {
return Err(DataFusionError::Execution(format!(
"Cannot cast a batch with {} columns to a schema with {} fields",
batch.num_columns(),
schema.fields().len()
)));
}
let columns = batch
.columns()
.iter()
.zip(schema.fields())
.map(|(column, field)| cast_column(column, field.data_type()))
.collect::<Result<Vec<_>, _>>()?;
let options = RecordBatchOptions::new().with_row_count(Some(batch.num_rows()));
RecordBatch::try_new_with_options(Arc::clone(schema), columns, &options).map_err(Into::into)
}
33 changes: 3 additions & 30 deletions crates/integrations/datafusion/src/physical_plan/scan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,11 @@
use std::sync::Arc;

use datafusion::arrow::array::BooleanArray;
use datafusion::arrow::compute::{cast, filter_record_batch};
use datafusion::arrow::compute::filter_record_batch;
use datafusion::arrow::datatypes::{
DataType as ArrowDataType, SchemaRef as ArrowSchemaRef, TimeUnit,
};
use datafusion::arrow::record_batch::{RecordBatch, RecordBatchOptions};
use datafusion::arrow::record_batch::RecordBatch;
use datafusion::common::stats::Precision;
use datafusion::common::tree_node::TreeNodeRecursion;
use datafusion::common::{ColumnStatistics, ScalarValue, Statistics};
Expand Down Expand Up @@ -64,33 +64,6 @@ use paimon::DataSplit;
use crate::error::to_datafusion_error;
use crate::filter_pushdown::{is_float_zero, scalar_to_datum};

fn to_datafusion_batch(batch: RecordBatch, schema: &ArrowSchemaRef) -> DFResult<RecordBatch> {
if batch.num_columns() != schema.fields().len() {
return Err(datafusion::error::DataFusionError::Execution(format!(
"Paimon reader returned {} columns for DataFusion schema with {} fields",
batch.num_columns(),
schema.fields().len()
)));
}

let row_count = batch.num_rows();
let columns = batch
.columns()
.iter()
.zip(schema.fields())
.map(|(column, field)| {
if column.data_type() == field.data_type() {
Ok(Arc::clone(column))
} else {
cast(column.as_ref(), field.data_type()).map_err(Into::into)
}
})
.collect::<DFResult<Vec<_>>>()?;
let options = RecordBatchOptions::new().with_row_count(Some(row_count));

RecordBatch::try_new_with_options(Arc::clone(schema), columns, &options).map_err(Into::into)
}

#[derive(Debug)]
struct DataFusionRowFilterFactory {
predicate: Arc<dyn PhysicalExpr>,
Expand Down Expand Up @@ -1125,7 +1098,7 @@ impl PaimonTableScan {
let stream = stream.map(move |result| {
let mut batch = result
.map_err(to_datafusion_error)
.and_then(|batch| to_datafusion_batch(batch, &batch_schema))?;
.and_then(|batch| super::cast_batch_to_schema(batch, &batch_schema))?;
// The decoder hook is an optimization and may be unavailable
// for a file/path. Retain every original live expression as
// the exact fallback; evaluating it on decoder survivors is
Expand Down
2 changes: 1 addition & 1 deletion crates/integrations/datafusion/src/procedures/compat.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ async fn validate_write_target(
Ident::with_quote('"', identifier.database()),
Ident::with_quote('"', identifier.object()),
]);
SQLContext::ensure_partition_command_target(&name, operation)?;
SQLContext::ensure_main_branch_base_table(&name, operation)?;
if let Some(sql_ctx) = sql_ctx {
sql_ctx.ensure_no_time_travel_for_write(operation)?;
sql_ctx
Expand Down
20 changes: 20 additions & 0 deletions crates/integrations/datafusion/src/relation_planner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,26 @@ use crate::catalog::{
};
use crate::table::{ObjectTableProvider, PaimonTableProvider};

#[derive(Debug)]
pub(crate) struct SnapshotRelationPlanner(
pub HashMap<String, (TableReference, Arc<dyn TableProvider>)>,
);

impl RelationPlanner for SnapshotRelationPlanner {
fn plan_relation(
&self,
relation: TableFactor,
_: &mut dyn RelationPlannerContext,
) -> DFResult<RelationPlanning> {
if let TableFactor::Table { name, .. } = &relation {
if let Some((table, provider)) = self.0.get(&name.to_string()) {
return plan_scan(relation, table.clone(), provider.clone());
}
}
Ok(RelationPlanning::Original(Box::new(relation)))
}
}

/// Resolves Paimon time travel, and registered engine snapshot IDs and timestamps.
#[derive(Debug)]
pub struct PaimonRelationPlanner;
Expand Down
Loading
Loading