From f78635d4d03da5996efd552c15d9e4c850063ae5 Mon Sep 17 00:00:00 2001 From: Nicholas Jiang Date: Sun, 22 Nov 2026 12:44:47 +0800 Subject: [PATCH] [datafusion] Support persistent CREATE TABLE AS SELECT Materializing a query as a Paimon table required an explicit schema and a separate INSERT. Once the catalog publishes a table, other clients can write to it, so dropping it after a failed write can discard their rows. - Infer the schema from the analyzed query, preserving nullability and geospatial metadata even for empty results. Apply options, comments and partition keys through CREATE TABLE validation, addressing #1425. - Resolve source tables and SQL functions in the session catalog and database at statement start. Populate from the same plan to preserve quoted identifiers and time-travel references across catalogs. - Check targets before source planning and honor IF NOT EXISTS. Preserve catalog lookup errors and use filesystem and REST metadata without opening table storage. Require atomic CTAS creation and support ordinary CREATE on storage without conditional publication. - Retain the table after a write failure and discard only owned, uncommitted files. Finish in-flight commits after cancellation and keep prepared files when the publication outcome is unknown. --- Cargo.lock | 1 + crates/integration_tests/Cargo.toml | 1 + crates/integrations/datafusion/README.md | 17 + .../src/format_partition_analyze.rs | 2 +- .../datafusion/src/format_partition_ddl.rs | 2 +- .../datafusion/src/format_partition_repair.rs | 2 +- .../datafusion/src/physical_plan/mod.rs | 91 + .../datafusion/src/physical_plan/scan.rs | 33 +- .../datafusion/src/procedures/compat.rs | 2 +- .../datafusion/src/relation_planner.rs | 37 +- .../datafusion/src/sql_context.rs | 1681 ++++++----------- .../datafusion/src/sql_context/ctas.rs | 285 +++ .../datafusion/tests/create_table_as.rs | 946 ++++++++++ .../tests/create_table_as/execution.rs | 726 +++++++ .../datafusion/tests/read_files_behavior.rs | 13 +- crates/paimon/src/catalog/filesystem.rs | 369 +++- crates/paimon/src/catalog/mod.rs | 24 + .../paimon/src/catalog/rest/rest_catalog.rs | 71 +- crates/paimon/src/io/file_io.rs | 89 +- crates/paimon/src/table/data_file_writer.rs | 8 +- crates/paimon/src/table/rest_env.rs | 7 +- crates/paimon/src/table/table_commit.rs | 4 +- crates/paimon/tests/mock_server.rs | 53 +- crates/paimon/tests/rest_catalog_test.rs | 73 +- docs/src/sql.md | 73 +- 25 files changed, 3350 insertions(+), 1260 deletions(-) create mode 100644 crates/integrations/datafusion/src/sql_context/ctas.rs create mode 100644 crates/integrations/datafusion/tests/create_table_as.rs create mode 100644 crates/integrations/datafusion/tests/create_table_as/execution.rs diff --git a/Cargo.lock b/Cargo.lock index d69d178c8..574a3db36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5081,6 +5081,7 @@ dependencies = [ "tar", "tempfile", "tokio", + "uuid", ] [[package]] diff --git a/crates/integration_tests/Cargo.toml b/crates/integration_tests/Cargo.toml index 75ec66175..b5e68ef8f 100644 --- a/crates/integration_tests/Cargo.toml +++ b/crates/integration_tests/Cargo.toml @@ -38,3 +38,4 @@ tar = "0.4" tempfile = "3" serde_json = "1" axum = { version = "0.7", features = ["macros", "tokio", "http1", "http2"] } +uuid = { version = "1", features = ["v4"] } diff --git a/crates/integrations/datafusion/README.md b/crates/integrations/datafusion/README.md index a91a968ab..c12d7e8dc 100644 --- a/crates/integrations/datafusion/README.md +++ b/crates/integrations/datafusion/README.md @@ -149,4 +149,21 @@ let view = ctx.sql("SELECT * FROM analytics_view").await?; let function = ctx.sql("SELECT plus_one(score) FROM scores").await?; ``` +## Persistent CREATE TABLE AS SELECT + +Use `CREATE TABLE AS SELECT` to materialize a query as a native Paimon table. +The query determines the destination schema. In this example, `scores` resolves +in the session's current catalog and database. + +```sql +CREATE TABLE paimon.reporting.results +WITH ('bucket' = '-1') +AS SELECT id, score FROM scores WHERE score > 10; +``` + +`IF NOT EXISTS` skips an existing target. If population fails after creation, +the destination remains in the catalog. For supported types, concurrent DDL, +cancellation and recovery after a commit error, see the +[SQL reference](../../../docs/src/sql.md#create-table-as-select). + See the [documentation](https://paimon.apache.org/docs/rust/datafusion/) for getting started guide and more details. diff --git a/crates/integrations/datafusion/src/format_partition_analyze.rs b/crates/integrations/datafusion/src/format_partition_analyze.rs index 01946d70c..99e97de3a 100644 --- a/crates/integrations/datafusion/src/format_partition_analyze.rs +++ b/crates/integrations/datafusion/src/format_partition_analyze.rs @@ -57,7 +57,7 @@ pub(crate) async fn execute_analyze( "ANALYZE TABLE requires COMPUTE STATISTICS".to_string(), )); } - SQLContext::ensure_partition_command_target(table_name, "ANALYZE TABLE")?; + SQLContext::ensure_main_branch_base_table(table_name, "ANALYZE TABLE")?; ctx.reject_temporary_write_target(table_name, "ANALYZE TABLE") .await?; let (catalog, _catalog_name, identifier) = ctx.resolve_catalog_and_table(table_name)?; diff --git a/crates/integrations/datafusion/src/format_partition_ddl.rs b/crates/integrations/datafusion/src/format_partition_ddl.rs index c2d4eeac9..33d39f587 100644 --- a/crates/integrations/datafusion/src/format_partition_ddl.rs +++ b/crates/integrations/datafusion/src/format_partition_ddl.rs @@ -112,7 +112,7 @@ pub(crate) async fn execute_show_partitions( show_partitions: &ShowPartitionsStatement, enable_ident_normalization: bool, ) -> DFResult { - SQLContext::ensure_partition_command_target(&show_partitions.table_name, "SHOW PARTITIONS")?; + SQLContext::ensure_main_branch_base_table(&show_partitions.table_name, "SHOW PARTITIONS")?; let (catalog, _catalog_name, identifier) = ctx.resolve_catalog_and_table(&show_partitions.table_name)?; let table = catalog diff --git a/crates/integrations/datafusion/src/format_partition_repair.rs b/crates/integrations/datafusion/src/format_partition_repair.rs index 1f018b382..78dcf44c2 100644 --- a/crates/integrations/datafusion/src/format_partition_repair.rs +++ b/crates/integrations/datafusion/src/format_partition_repair.rs @@ -36,7 +36,7 @@ pub(crate) async fn execute_msck(ctx: &SQLContext, msck: &Msck) -> DFResult Result { + use DataType::*; + if data.child_data().is_empty() { + return Ok(data); + } + let children = data + .child_data() + .iter() + .cloned() + .map(normalize_map_ordering) + .collect::, _>>()?; + let field = |original: &FieldRef, index: usize| { + Arc::new( + original + .as_ref() + .clone() + .with_data_type(children[index].data_type().clone()), + ) + }; + let data_type = match data.data_type() { + Map(entries, _) => Map(field(entries, 0), false), + Struct(fields) => Struct( + fields + .iter() + .enumerate() + .map(|(i, f)| field(f, i)) + .collect(), + ), + List(f) => List(field(f, 0)), + LargeList(f) => LargeList(field(f, 0)), + FixedSizeList(f, size) => FixedSizeList(field(f, 0), *size), + Dictionary(key, _) => Dictionary(key.clone(), Box::new(children[0].data_type().clone())), + _ => return Ok(data), + }; + if &data_type == data.data_type() { + return Ok(data); + } + data.into_builder() + .data_type(data_type) + .child_data(children) + .build() +} + +fn cast_column(column: &ArrayRef, target: &DataType) -> Result { + if column.data_type() == target { + return Ok(Arc::clone(column)); + } + if !can_cast_types(column.data_type(), target) { + let data = normalize_map_ordering(column.to_data())?; + if data.data_type() != column.data_type() { + return cast(&make_array(data), target); + } + } + cast(column, target) +} + +pub(crate) fn cast_batch_to_schema( + batch: RecordBatch, + schema: &SchemaRef, +) -> DFResult { + if batch.schema_ref() == schema { + return Ok(batch); + } + if batch.num_columns() != schema.fields().len() { + return Err(DataFusionError::Execution(format!( + "Cannot cast a batch with {} columns to a schema with {} fields", + batch.num_columns(), + schema.fields().len() + ))); + } + let columns = batch + .columns() + .iter() + .zip(schema.fields()) + .map(|(column, field)| cast_column(column, field.data_type())) + .collect::, _>>()?; + let options = RecordBatchOptions::new().with_row_count(Some(batch.num_rows())); + RecordBatch::try_new_with_options(Arc::clone(schema), columns, &options).map_err(Into::into) +} diff --git a/crates/integrations/datafusion/src/physical_plan/scan.rs b/crates/integrations/datafusion/src/physical_plan/scan.rs index 0b60f8073..fce9e6ffa 100644 --- a/crates/integrations/datafusion/src/physical_plan/scan.rs +++ b/crates/integrations/datafusion/src/physical_plan/scan.rs @@ -18,11 +18,11 @@ use std::sync::Arc; use datafusion::arrow::array::BooleanArray; -use datafusion::arrow::compute::{cast, filter_record_batch}; +use datafusion::arrow::compute::filter_record_batch; use datafusion::arrow::datatypes::{ DataType as ArrowDataType, SchemaRef as ArrowSchemaRef, TimeUnit, }; -use datafusion::arrow::record_batch::{RecordBatch, RecordBatchOptions}; +use datafusion::arrow::record_batch::RecordBatch; use datafusion::common::stats::Precision; use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{ColumnStatistics, ScalarValue, Statistics}; @@ -64,33 +64,6 @@ use paimon::DataSplit; use crate::error::to_datafusion_error; use crate::filter_pushdown::{is_float_zero, scalar_to_datum}; -fn to_datafusion_batch(batch: RecordBatch, schema: &ArrowSchemaRef) -> DFResult { - if batch.num_columns() != schema.fields().len() { - return Err(datafusion::error::DataFusionError::Execution(format!( - "Paimon reader returned {} columns for DataFusion schema with {} fields", - batch.num_columns(), - schema.fields().len() - ))); - } - - let row_count = batch.num_rows(); - let columns = batch - .columns() - .iter() - .zip(schema.fields()) - .map(|(column, field)| { - if column.data_type() == field.data_type() { - Ok(Arc::clone(column)) - } else { - cast(column.as_ref(), field.data_type()).map_err(Into::into) - } - }) - .collect::>>()?; - let options = RecordBatchOptions::new().with_row_count(Some(row_count)); - - RecordBatch::try_new_with_options(Arc::clone(schema), columns, &options).map_err(Into::into) -} - #[derive(Debug)] struct DataFusionRowFilterFactory { predicate: Arc, @@ -1125,7 +1098,7 @@ impl PaimonTableScan { let stream = stream.map(move |result| { let mut batch = result .map_err(to_datafusion_error) - .and_then(|batch| to_datafusion_batch(batch, &batch_schema))?; + .and_then(|batch| super::cast_batch_to_schema(batch, &batch_schema))?; // The decoder hook is an optimization and may be unavailable // for a file/path. Retain every original live expression as // the exact fallback; evaluating it on decoder survivors is diff --git a/crates/integrations/datafusion/src/procedures/compat.rs b/crates/integrations/datafusion/src/procedures/compat.rs index d4569fff5..5eed29b48 100644 --- a/crates/integrations/datafusion/src/procedures/compat.rs +++ b/crates/integrations/datafusion/src/procedures/compat.rs @@ -58,7 +58,7 @@ async fn validate_write_target( Ident::with_quote('"', identifier.database()), Ident::with_quote('"', identifier.object()), ]); - SQLContext::ensure_partition_command_target(&name, operation)?; + SQLContext::ensure_main_branch_base_table(&name, operation)?; if let Some(sql_ctx) = sql_ctx { sql_ctx.ensure_no_time_travel_for_write(operation)?; sql_ctx diff --git a/crates/integrations/datafusion/src/relation_planner.rs b/crates/integrations/datafusion/src/relation_planner.rs index 10b2679db..deb7a8ae3 100644 --- a/crates/integrations/datafusion/src/relation_planner.rs +++ b/crates/integrations/datafusion/src/relation_planner.rs @@ -39,6 +39,31 @@ use crate::catalog::{ }; use crate::table::{ObjectTableProvider, PaimonTableProvider}; +/// Resolves statement-local TABLE bindings to snapshot scans with the original table qualifier. +#[derive(Debug)] +pub(crate) struct SnapshotRelationPlanner( + pub HashMap)>, +); + +impl RelationPlanner for SnapshotRelationPlanner { + fn plan_relation( + &self, + relation: TableFactor, + _: &mut dyn RelationPlannerContext, + ) -> DFResult { + if let TableFactor::TableFunction { + expr: ast::Expr::Identifier(binding), + alias, + } = &relation + { + if let Some((table, provider)) = self.0.get(&binding.value) { + return plan_scan(alias.clone(), table.clone(), provider.clone()); + } + } + Ok(RelationPlanning::Original(Box::new(relation))) + } +} + /// Resolves Paimon time travel, and registered engine snapshot IDs and timestamps. #[derive(Debug)] pub struct PaimonRelationPlanner; @@ -64,6 +89,7 @@ impl RelationPlanner for PaimonRelationPlanner { let TableFactor::Table { ref name, ref version, + ref alias, .. } = relation else { @@ -102,7 +128,7 @@ impl RelationPlanner for PaimonRelationPlanner { async move { routed.at_time_travel(travel).await }, "engine time travel resolution thread panicked", )?; - return plan_scan(relation, table_ref, new_provider); + return plan_scan(alias.clone(), table_ref, new_provider); } let Some(paimon_provider) = provider.downcast_ref::() else { @@ -140,18 +166,15 @@ impl RelationPlanner for PaimonRelationPlanner { ) .map_err(crate::to_datafusion_error)?; let new_provider = PaimonTableProvider::try_new(new_table)?; - plan_scan(relation, table_ref, Arc::new(new_provider)) + plan_scan(alias.clone(), table_ref, Arc::new(new_provider)) } } fn plan_scan( - relation: TableFactor, + alias: Option, table_ref: TableReference, provider: Arc, ) -> DFResult { - let TableFactor::Table { alias, .. } = relation else { - unreachable!() - }; let plan = LogicalPlanBuilder::scan(table_ref, provider_as_source(provider), None)?.build()?; Ok(RelationPlanning::Planned(Box::new(PlannedRelation::new( plan, alias, @@ -245,7 +268,7 @@ fn timestamp_as_of_millis(expr: &ast::Expr) -> DFResult { /// Parse a timestamp string to milliseconds since epoch (using local timezone). /// /// Matches Java Paimon's behavior which uses `TimeZone.getDefault()`. -fn parse_timestamp_to_millis(ts: &str) -> DFResult { +pub(crate) fn parse_timestamp_to_millis(ts: &str) -> DFResult { use chrono::{Local, NaiveDateTime, TimeZone}; let naive = NaiveDateTime::parse_from_str(ts, "%Y-%m-%d %H:%M:%S").map_err(|e| { diff --git a/crates/integrations/datafusion/src/sql_context.rs b/crates/integrations/datafusion/src/sql_context.rs index a8a98b7d3..dba92461a 100644 --- a/crates/integrations/datafusion/src/sql_context.rs +++ b/crates/integrations/datafusion/src/sql_context.rs @@ -28,6 +28,7 @@ //! - `DROP DATABASE [IF EXISTS] [catalog.]db [CASCADE]` //! - `USE [catalog.]db` //! - `CREATE TABLE db.t (col TYPE, ..., PRIMARY KEY (col, ...)) [PARTITIONED BY (col, ...)] [WITH ('key' = 'val')]` +//! - `CREATE TABLE [IF NOT EXISTS] db.t [PARTITIONED BY (col, ...)] [WITH ('key' = 'val')] AS query` //! - `ALTER TABLE db.t ADD COLUMN col TYPE` //! - `ALTER TABLE db.t DROP COLUMN col` //! - `ALTER TABLE db.t RENAME COLUMN old TO new` @@ -55,6 +56,7 @@ use datafusion::arrow::array::{ use datafusion::arrow::compute::cast; use datafusion::arrow::datatypes::{DataType as ArrowDataType, Field, Schema}; use datafusion::arrow::record_batch::{RecordBatch, RecordBatchOptions}; +use datafusion::common::config::CatalogOptions; use datafusion::common::tree_node::{TreeNode, TreeNodeRecursion}; use datafusion::common::{plan_datafusion_err, DFSchema, ScalarValue, TableReference}; use datafusion::datasource::{MemTable, TableProvider}; @@ -78,7 +80,7 @@ use datafusion::sql::sqlparser::ast::{ use datafusion::sql::sqlparser::dialect::GenericDialect; use datafusion::sql::sqlparser::keywords::Keyword; use datafusion::sql::sqlparser::parser::Parser; -use datafusion::sql::sqlparser::tokenizer::{Token, Tokenizer}; +use datafusion::sql::sqlparser::tokenizer::{Location, Token, Tokenizer}; use datafusion_spark::SessionStateBuilderSpark; use futures::StreamExt; use paimon::catalog::{parse_object_name, Catalog, Identifier, ParsedObjectName}; @@ -94,6 +96,8 @@ use crate::error::to_datafusion_error; use crate::table::PaimonTableProvider; use crate::{BlobReaderRegistry, DynamicOptions}; +mod ctas; + /// A SQL context that supports registering multiple Paimon catalogs and executing SQL. /// /// # Example @@ -490,17 +494,17 @@ impl SQLContext { /// Execute a SQL statement with options for statements delegated to DataFusion. pub async fn sql_with_options(&self, sql: &str, options: SQLOptions) -> DFResult { - let is_create_table = looks_like_create_table(sql); - let enable_ident_normalization = self.ctx.enable_ident_normalization(); - let (rewritten_sql, partition_keys) = if is_create_table { - extract_partition_by(sql, enable_ident_normalization)? - } else { - (sql.to_string(), vec![]) - }; - if contains_time_travel_keyword(&rewritten_sql) { - // Time-travel queries are not DDL; skip our own parsing and handle directly. - return self.handle_time_travel_query(&rewritten_sql, options).await; - } + // Box the dispatcher so its size does not propagate to callers' futures. + Box::pin(self.execute_statement(sql, options)).await + } + + async fn execute_statement(&self, sql: &str, options: SQLOptions) -> DFResult { + let state = self.ctx.state(); + let enable_ident_normalization = + state.config_options().sql_parser.enable_ident_normalization; + let (mut rewritten_sql, partition_keys) = + extract_partition_by(sql, enable_ident_normalization)?; + let time_travel_sources = mask_time_travel_sources(&mut rewritten_sql); if let Some(show_partitions) = crate::format_partition_ddl::parse_show_partitions(&rewritten_sql)? { @@ -526,33 +530,50 @@ impl SQLContext { )); } + if !time_travel_sources.is_empty() { + validate_time_travel_sources(&statements[0], &time_travel_sources)?; + } + if let Statement::CreateTable(ct) = &mut statements[0] { + if let Some(query) = ct.query.take_if(|_| !ct.temporary) { + return Box::pin(self.handle_create_table_as( + ct, + query, + partition_keys, + state, + options, + &time_travel_sources, + )) + .await; + } + } + if !time_travel_sources.is_empty() + || matches!( + statements[0], + Statement::Query(_) | Statement::Explain { .. } + ) + { + return Box::pin(self.plan_query( + state, + statements.remove(0), + options, + &time_travel_sources, + )) + .await; + } + let rewritten; let stored_definition = matches!(&statements[0], Statement::CreateView(view) if view.temporary || self.is_paimon_catalog_ref(&TableReference::from(view.name.to_string().as_str()))) || matches!(&statements[0], Statement::CreateFunction(_)); let changed = !stored_definition && { - let binary_changed = crate::binary_cast::rewrite(&mut statements[0])?; - let regex_changed = crate::regex_operators::rewrite(&mut statements[0]); - let lateral_changed = crate::lateral_views::rewrite(&mut statements[0])?; - let compat_changed = - crate::expression_compat::rewrite(&mut statements[0], &self.ctx.state())?; - let state = self.ctx.state(); - // Queries and temporary view bodies resolve after SQL function expansion below. - let max_pt_changed = if matches!( - statements[0], - Statement::Query(_) | Statement::Explain { .. } - ) || matches!(&statements[0], Statement::CreateView(view) if view.temporary) - { - false - } else { - Box::pin(crate::partition_functions::rewrite( - &mut statements[0], - &state, - &self.current_catalog_name(), - &state.config_options().catalog.default_schema, - )) - .await? - }; - binary_changed || regex_changed || lateral_changed || compat_changed || max_pt_changed + let expressions_changed = Self::rewrite_expressions(&mut statements[0], &state)?; + let partitions_changed = Box::pin(crate::partition_functions::rewrite( + &mut statements[0], + &state, + &state.config_options().catalog.default_catalog, + &state.config_options().catalog.default_schema, + )) + .await?; + expressions_changed || partitions_changed }; let sql = if changed || vector_changed { rewritten = statements[0].to_string(); @@ -936,127 +957,65 @@ impl SQLContext { )) .await } - Statement::Query(_) | Statement::Explain { .. } => { - let mut statement = statements[0].clone(); - if let Statement::Explain { - statement: inner, .. - } = &mut statement - { - if let Statement::Merge(merge) = inner.as_mut() { - *merge = self.normalize_merge_into(merge).await?; - } - } - let current_catalog = self.current_catalog_name(); - let current_database = self - .ctx - .state() - .config_options() - .catalog - .default_schema - .clone(); - let expanded = crate::sql_function::expand_statement( - &self.ctx.state(), - statement, - &self.catalogs, - ¤t_catalog, - ¤t_database, - ) - .await?; - crate::read_files::sql(&self.ctx, &expanded.to_string(), options).await - } _ => crate::read_files::sql(&self.ctx, sql, options).await, } } - /// Rewrite time-travel references to temporary snapshot providers. - /// The tracker removes registrations after planning. - async fn handle_time_travel_query( + fn rewrite_expressions(statement: &mut Statement, state: &SessionState) -> DFResult { + let binary = crate::binary_cast::rewrite(statement)?; + let regex = crate::regex_operators::rewrite(statement); + let lateral = crate::lateral_views::rewrite(statement)?; + let compat = crate::expression_compat::rewrite(statement, state)?; + Ok(binary || regex || lateral || compat) + } + + async fn plan_query( &self, - sql: &str, - sql_options: SQLOptions, + mut state: SessionState, + mut statement: Statement, + options: SQLOptions, + sources: &[TimeTravelSource], ) -> DFResult { - let mut tracker = crate::merge_into::TempTableTracker::new(self); - - let version_clauses = extract_all_version_as_of(sql); - let timestamp_clauses = extract_all_timestamp_as_of(sql); - - if version_clauses.is_empty() && timestamp_clauses.is_empty() { - return Err(DataFusionError::Plan( - "Failed to parse time-travel clause in SQL".to_string(), - )); - } - - let mut replacements: Vec<((usize, usize), String)> = Vec::new(); - - for info in &version_clauses { - let table_ref: datafusion::common::TableReference = info.table_name.as_str().into(); - let (catalog, catalog_name, identifier) = - self.resolve_table_name_from_ref(&table_ref)?; - let provider = self - .time_travel_provider( - &catalog, - &catalog_name, - &identifier, - TimeTravelClause::Version(info.version.clone()), - ) - .await?; - - let uuid_name = format!("__paimon_tt_{}", uuid::Uuid::new_v4().as_simple()); - self.register_temp_table(uuid_name.as_str(), provider)?; - tracker.register(&uuid_name); - replacements.push((info.clause_range, uuid_name)); + Self::rewrite_expressions(&mut statement, &state)?; + if let Statement::Explain { + statement: inner, .. + } = &mut statement + { + if let Statement::Merge(merge) = inner.as_mut() { + *merge = self.normalize_merge_into(merge).await?; + } } - - for info in ×tamp_clauses { - let table_ref: datafusion::common::TableReference = info.table_name.as_str().into(); + let session_catalog = &state.config_options().catalog; + let mut bindings = HashMap::new(); + for source in sources { + let table = object_name_to_table_reference( + source.table.clone(), + state.config_options().sql_parser.enable_ident_normalization, + )?; let (catalog, catalog_name, identifier) = - self.resolve_table_name_from_ref(&table_ref)?; - let millis = Self::parse_timestamp_to_millis(&info.timestamp)?; + self.resolve_table_reference(&table, session_catalog)?; let provider = self - .time_travel_provider( - &catalog, - &catalog_name, - &identifier, - TimeTravelClause::TimestampMillis(millis), - ) + .time_travel_provider(&catalog, &catalog_name, &identifier, &source.clause) .await?; - - let uuid_name = format!("__paimon_tt_{}", uuid::Uuid::new_v4().as_simple()); - self.register_temp_table(uuid_name.as_str(), provider)?; - tracker.register(&uuid_name); - replacements.push((info.clause_range, uuid_name)); - } - - // Replace from right to left to preserve byte offsets. - replacements.sort_by_key(|r| std::cmp::Reverse(r.0 .0)); - - let mut rewritten_sql = sql.to_string(); - for ((start, end), uuid_name) in &replacements { - rewritten_sql = format!( - "{}{}{}", - &rewritten_sql[..*start], - uuid_name, - &rewritten_sql[*end..] - ); + bindings.insert(source.binding.clone(), (table, provider)); } - - let current_catalog = self.current_catalog_name(); - let current_database = self - .ctx - .state() - .config_options() - .catalog - .default_schema - .clone(); - let expanded = crate::sql_function::expand_sql( - &self.ctx.state(), - &rewritten_sql, + let expanded = crate::sql_function::expand_statement( + &state, + statement, &self.catalogs, - ¤t_catalog, - ¤t_database, + &session_catalog.default_catalog, + &session_catalog.default_schema, ) .await?; - crate::read_files::sql(&self.ctx, &expanded, sql_options).await + let sql = expanded.to_string(); + if !bindings.is_empty() { + state.register_relation_planner(Arc::new( + crate::relation_planner::SnapshotRelationPlanner(bindings), + ))?; + } + // USE can change the shared session while catalog lookups are in flight. + let context = SessionContext::new_with_state(state); + crate::read_files::sql(&context, &sql, options).await } async fn time_travel_provider( @@ -1064,9 +1023,10 @@ impl SQLContext { catalog: &Arc, catalog_name: &str, identifier: &Identifier, - clause: TimeTravelClause, + clause: &TimeTravelClause, ) -> DFResult> { use crate::catalog::EngineTimeTravel; + use crate::relation_planner::parse_timestamp_to_millis; use paimon::spec::{SCAN_TIMESTAMP_MILLIS_OPTION, SCAN_VERSION_OPTION}; let parsed = identifier @@ -1096,10 +1056,10 @@ impl SQLContext { } let travel = match clause { TimeTravelClause::Version(version) => { - EngineTimeTravel::Snapshot(crate::catalog::parse_engine_snapshot_id(&version)?) + EngineTimeTravel::Snapshot(crate::catalog::parse_engine_snapshot_id(version)?) } - TimeTravelClause::TimestampMillis(millis) => { - EngineTimeTravel::TimestampMillis(millis) + TimeTravelClause::Timestamp(timestamp) => { + EngineTimeTravel::TimestampMillis(parse_timestamp_to_millis(timestamp)?) } }; let provider = self @@ -1126,10 +1086,11 @@ impl SQLContext { let (table, base, system_name) = crate::table_loader::loaded_table_for_read(loaded, identifier).await?; let (option, value) = match clause { - TimeTravelClause::Version(version) => (SCAN_VERSION_OPTION, version), - TimeTravelClause::TimestampMillis(millis) => { - (SCAN_TIMESTAMP_MILLIS_OPTION, millis.to_string()) - } + TimeTravelClause::Version(version) => (SCAN_VERSION_OPTION, version.clone()), + TimeTravelClause::Timestamp(timestamp) => ( + SCAN_TIMESTAMP_MILLIS_OPTION, + parse_timestamp_to_millis(timestamp)?.to_string(), + ), }; let mut options = self.dynamic_options.read().unwrap().clone(); options.insert(option.to_string(), value); @@ -1155,68 +1116,22 @@ impl SQLContext { } } - /// Parse a timestamp string to milliseconds since epoch (using local timezone). - fn parse_timestamp_to_millis(ts: &str) -> DFResult { - use chrono::{Local, NaiveDateTime, TimeZone}; - - let naive = NaiveDateTime::parse_from_str(ts, "%Y-%m-%d %H:%M:%S").map_err(|e| { - DataFusionError::Plan(format!( - "Cannot parse time travel timestamp '{ts}': {e}. Expected format: YYYY-MM-DD HH:MM:SS" - )) - })?; - let local = Local.from_local_datetime(&naive).single().ok_or_else(|| { - DataFusionError::Plan(format!("Ambiguous or invalid local time: '{ts}'")) - })?; - Ok(local.timestamp_millis()) - } - - /// Resolve a TableReference to (catalog, catalog_name, Identifier). - fn resolve_table_name_from_ref( + fn resolve_table_reference( &self, - table_ref: &datafusion::common::TableReference, + table: &TableReference, + defaults: &CatalogOptions, ) -> DFResult<(Arc, String, Identifier)> { - match table_ref { - datafusion::common::TableReference::Full { - catalog, - schema, - table, - } => { - let catalog_arc = self - .catalogs - .get(catalog.as_ref()) - .ok_or_else(|| DataFusionError::Plan(format!("Unknown catalog '{catalog}'")))?; - Ok(( - catalog_arc.clone(), - catalog.to_string(), - Identifier::new(schema.as_ref(), table.as_ref()), - )) - } - datafusion::common::TableReference::Partial { schema, table } => { - let catalog = self.current_catalog()?; - let catalog_name = self.current_catalog_name(); - Ok(( - catalog, - catalog_name, - Identifier::new(schema.as_ref(), table.as_ref()), - )) - } - datafusion::common::TableReference::Bare { table } => { - let catalog = self.current_catalog()?; - let catalog_name = self.current_catalog_name(); - let default_schema = self - .ctx - .state() - .config_options() - .catalog - .default_schema - .clone(); - Ok(( - catalog, - catalog_name, - Identifier::new(default_schema, table.as_ref()), - )) - } - } + let catalog_name = table.catalog().unwrap_or(&defaults.default_catalog); + let catalog = self + .catalogs + .get(catalog_name) + .ok_or_else(|| plan_datafusion_err!("Unknown catalog '{catalog_name}'"))?; + let database = table.schema().unwrap_or(&defaults.default_schema); + Ok(( + catalog.clone(), + catalog_name.to_string(), + Identifier::new(database, table.table()), + )) } async fn handle_create_table( @@ -1236,73 +1151,14 @@ impl SQLContext { "LOCATION is not supported for Paimon tables. Table path is determined by the catalog warehouse.".to_string(), )); } - if ct.query.is_some() { - return Err(DataFusionError::Plan( - "CREATE TABLE AS SELECT is not yet supported for Paimon tables.".to_string(), - )); - } - let identifier = self.resolve_table_name(&ct.name)?; - - let mut builder = paimon::spec::Schema::builder(); - let mut comments = Vec::new(); - if let Some(comment) = &ct.comment { - comments.push(comment); - } - let options = match &ct.table_options { - CreateTableOptions::With(options) - | CreateTableOptions::Options(options) - | CreateTableOptions::TableProperties(options) - | CreateTableOptions::Plain(options) => options.as_slice(), - CreateTableOptions::None => &[], - }; - for option in options { - if let SqlOption::Comment(comment) = option { - comments.push(comment); - } - } - if comments.len() > 1 { - return Err(DataFusionError::Plan("Duplicate table COMMENT".into())); - } - if let Some(CommentDef::WithEq(comment) | CommentDef::WithoutEq(comment)) = - comments.first().copied() - { - builder = builder.comment(Some(comment.clone())); - } - let table_options = extract_options(&ct.table_options)?; - let column_names: Vec = ct - .columns - .iter() - .map(|column| normalize_schema_identifier(&column.name, enable_ident_normalization)) - .collect(); - - // Columns - for (col, column_name) in ct.columns.iter().zip(&column_names) { - let paimon_type = column_def_to_paimon_type(col)?; - let comment = column_def_comment(col); - builder = builder.column_with_description(column_name.clone(), paimon_type, comment); - } - - builder = builder.primary_key(extract_primary_keys(ct, enable_ident_normalization)?); - - // Partition keys (extracted and validated before parsing) - if !partition_keys.is_empty() { - for pk in &partition_keys { - if !column_names.contains(pk) { - return Err(DataFusionError::Plan(format!( - "PARTITIONED BY column '{pk}' is not defined in the table" - ))); - } - } - builder = builder.partition_keys(partition_keys); - } - - // Table options from WITH ('key' = 'value', ...) - for (k, v) in table_options { - builder = builder.option(k, v); - } - - let schema = builder.build().map_err(to_datafusion_error)?; + let schema = create_table_schema( + ct, + partition_keys, + enable_ident_normalization, + None, + extract_options(&ct.table_options)?, + )?; catalog .create_table(&identifier, schema, ct.if_not_exists) @@ -1752,7 +1608,7 @@ impl SQLContext { ) }); if has_partition_operation { - Self::ensure_partition_command_target(name, "ALTER TABLE")?; + Self::ensure_main_branch_base_table(name, "ALTER TABLE")?; } else { Self::ensure_main_branch_write_target(name, "ALTER TABLE")?; } @@ -2874,7 +2730,7 @@ impl SQLContext { Ok(()) } - pub(crate) fn ensure_partition_command_target( + pub(crate) fn ensure_main_branch_base_table( name: &ObjectName, operation: &str, ) -> DFResult<()> { @@ -2909,6 +2765,84 @@ impl SQLContext { } } +fn create_table_schema( + ct: &CreateTable, + partition_keys: Vec, + enable_ident_normalization: bool, + inferred_schema: Option<&Schema>, + table_options: HashMap, +) -> DFResult { + let mut builder = paimon::spec::Schema::builder(); + let mut comments = Vec::new(); + if let Some(comment) = &ct.comment { + comments.push(comment); + } + let options = match &ct.table_options { + CreateTableOptions::With(options) + | CreateTableOptions::Options(options) + | CreateTableOptions::TableProperties(options) + | CreateTableOptions::Plain(options) => options.as_slice(), + CreateTableOptions::None => &[], + }; + for option in options { + if let SqlOption::Comment(comment) = option { + comments.push(comment); + } + } + if comments.len() > 1 { + return Err(DataFusionError::Plan("Duplicate table COMMENT".into())); + } + if let Some(CommentDef::WithEq(comment) | CommentDef::WithoutEq(comment)) = + comments.first().copied() + { + builder = builder.comment(Some(comment.clone())); + } + let column_names: Vec = if let Some(schema) = inferred_schema { + for field in schema.fields() { + let data_type = + paimon::arrow::arrow_field_to_paimon_type(field).map_err(to_datafusion_error)?; + builder = builder.column(field.name(), data_type); + } + schema + .fields() + .iter() + .map(|field| field.name().clone()) + .collect() + } else { + let names: Vec = ct + .columns + .iter() + .map(|column| normalize_schema_identifier(&column.name, enable_ident_normalization)) + .collect(); + for (col, name) in ct.columns.iter().zip(&names) { + builder = builder.column_with_description( + name.clone(), + column_def_to_paimon_type(col)?, + column_def_comment(col), + ); + } + names + }; + + builder = builder.primary_key(extract_primary_keys(ct, enable_ident_normalization)?); + + if !partition_keys.is_empty() { + for pk in &partition_keys { + if !column_names.contains(pk) { + return Err(DataFusionError::Plan(format!( + "PARTITIONED BY column '{pk}' is not defined in the table" + ))); + } + } + builder = builder.partition_keys(partition_keys); + } + + builder + .options(table_options) + .build() + .map_err(to_datafusion_error) +} + fn typed_view_query(view: &paimon::catalog::View) -> DFResult { let fields = view.schema().fields(); if fields.is_empty() { @@ -3157,6 +3091,7 @@ fn parse_sql_statements(sql: &str) -> DFResult<(Vec, bool)> { let significant = tokens .iter() .enumerate() + .skip_while(|(_, token)| matches!(token.token, Token::Whitespace(_) | Token::SemiColon)) .filter_map(|(index, token)| match &token.token { Token::Whitespace(_) => None, Token::Word(word) => Some((index, word.keyword)), @@ -3199,6 +3134,19 @@ fn parse_sql_statements(sql: &str) -> DFResult<(Vec, bool)> { let (mut statements, vector_changed) = crate::vector_type::parse_statements(tokens) .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; crate::geospatial::quote_cast_types(&mut statements); + // sqlparser can omit OR ALTER and earlier table-options clauses from the AST. + if matches!(statements.first(), Some(Statement::CreateTable(ct)) if !ct.temporary && ct.query.is_some()) + { + if !matches!( + significant.as_slice(), + [(_, Keyword::CREATE), (_, Keyword::TABLE), ..] + ) { + return Err(DataFusionError::NotImplemented( + "CREATE TABLE AS SELECT does not support modifiers before TABLE".into(), + )); + } + validate_ctas_option_clauses(sql)?; + } if create_function_if_not_exists { let Some(Statement::CreateFunction(create_function)) = statements.first_mut() else { return Err(DataFusionError::Plan( @@ -3322,284 +3270,144 @@ fn validate_persistent_create_view(create_view: &CreateView) -> DFResult<()> { Ok(()) } -/// Quick check whether the SQL looks like a CREATE TABLE statement. -/// Skips leading whitespace, `--` line comments, and `/* */` block comments. -fn looks_like_create_table(sql: &str) -> bool { - let bytes = sql.as_bytes(); - let len = bytes.len(); - let mut i = 0; - // Skip leading whitespace and comments - loop { - while i < len && bytes[i].is_ascii_whitespace() { - i += 1; - } - if i + 1 < len && bytes[i] == b'-' && bytes[i + 1] == b'-' { - i += 2; - while i < len && bytes[i] != b'\n' { - i += 1; - } - continue; - } - if i + 1 < len && bytes[i] == b'/' && bytes[i + 1] == b'*' { - i += 2; - while i + 1 < len { - if bytes[i] == b'*' && bytes[i + 1] == b'/' { - i += 2; - break; - } - i += 1; - } - continue; - } - break; - } - // Match "CREATE" then whitespace then optional "TEMPORARY"/"TEMP" then "TABLE" (all ASCII, byte-safe) - if i + 6 > len || !bytes[i..i + 6].eq_ignore_ascii_case(b"CREATE") { - return false; - } - i += 6; - if i >= len || !bytes[i].is_ascii_whitespace() { - return false; - } - while i < len && bytes[i].is_ascii_whitespace() { - i += 1; - } - // Skip optional TEMPORARY or TEMP keyword - if i + 9 <= len && bytes[i..i + 9].eq_ignore_ascii_case(b"TEMPORARY") { - i += 9; - while i < len && bytes[i].is_ascii_whitespace() { - i += 1; - } - } else if i + 4 <= len && bytes[i..i + 4].eq_ignore_ascii_case(b"TEMP") { - i += 4; - while i < len && bytes[i].is_ascii_whitespace() { - i += 1; - } - } - // After optional TEMPORARY/TEMP, reject CREATE TEMPORARY VIEW / CREATE TEMP VIEW - if i + 4 <= len && bytes[i..i + 4].eq_ignore_ascii_case(b"VIEW") { - return false; - } - i + 5 <= len && bytes[i..i + 5].eq_ignore_ascii_case(b"TABLE") -} - -/// Find `PARTITIONED BY` keyword position, skipping string literals and comments. -fn find_partitioned_by(sql: &str) -> Option<(usize, usize)> { - let bytes = sql.as_bytes(); - let len = bytes.len(); - let mut i = 0; - while i < len { - match bytes[i] { - b'\'' => { - i += 1; - while i < len { - if bytes[i] == b'\'' { - i += 1; - if i < len && bytes[i] == b'\'' { - i += 1; - } else { - break; - } - } else { - i += 1; - } - } - } - b'-' if i + 1 < len && bytes[i + 1] == b'-' => { - i += 2; - while i < len && bytes[i] != b'\n' { - i += 1; - } - } - b'/' if i + 1 < len && bytes[i + 1] == b'*' => { - i += 2; - while i + 1 < len { - if bytes[i] == b'*' && bytes[i + 1] == b'/' { - i += 2; - break; - } - i += 1; - } - } - b if b.is_ascii_alphabetic() && i + 11 <= len => { - if bytes[i..i + 11].eq_ignore_ascii_case(b"PARTITIONED") { - let rest = &bytes[i + 11..]; - let ws = rest.iter().take_while(|b| b.is_ascii_whitespace()).count(); - if ws > 0 - && i + 11 + ws + 2 <= len - && rest[ws..ws + 2].eq_ignore_ascii_case(b"BY") - { - let by_end = i + 11 + ws + 2; - return Some((i, by_end)); - } - } - i += 1; - } - _ => { - i += 1; - } +fn create_table_parser(sql: &str) -> Option> { + if !matches!( + sql.trim_start().as_bytes().first(), + Some(b'c' | b'C' | b'-' | b'/' | b';') + ) { + return None; + } + let mut parser = Parser::new(&GenericDialect).try_with_sql(sql).ok()?; + while parser.consume_token(&Token::SemiColon) {} + if !parser.parse_keyword(Keyword::CREATE) { + return None; + } + while let Token::Word(word) = parser.next_token().token { + match word.keyword { + Keyword::TABLE => return Some(parser), + Keyword::OR + | Keyword::REPLACE + | Keyword::ALTER + | Keyword::TEMP + | Keyword::TEMPORARY + | Keyword::EXTERNAL + | Keyword::GLOBAL + | Keyword::LOCAL + | Keyword::TRANSIENT + | Keyword::VOLATILE + | Keyword::DYNAMIC + | Keyword::ICEBERG + | Keyword::SNAPSHOT => {} + _ => break, } } None } -/// Parse a single partition column token, handling quoted identifiers. -fn parse_partition_column(token: &str, enable_ident_normalization: bool) -> DFResult { - let trimmed = token.trim(); - if trimmed.is_empty() { - return Err(DataFusionError::Plan( - "Empty column name in PARTITIONED BY".to_string(), - )); - } - - let first = trimmed.as_bytes()[0]; - if first == b'"' || first == b'`' { - let mut value = String::new(); - let mut end = None; - let mut chars = trimmed[1..].char_indices().peekable(); - while let Some((idx, ch)) = chars.next() { - if ch == first as char { - if chars.peek().is_some_and(|(_, next)| *next == first as char) { - value.push(ch); - chars.next(); - } else { - end = Some(1 + idx + ch.len_utf8()); - break; +fn validate_ctas_option_clauses(sql: &str) -> DFResult<()> { + let Some(mut parser) = create_table_parser(sql) else { + return Ok(()); + }; + let _ = parser.parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]); + parser.parse_object_name(false)?; + let mut depth = 0; + let mut seen_options = false; + loop { + match parser.next_token().token { + Token::EOF => return Ok(()), + Token::LParen => depth += 1, + Token::RParen => depth -= 1, + Token::Word(word) if depth == 0 => match word.keyword { + Keyword::AS => return Ok(()), + Keyword::WITH | Keyword::TBLPROPERTIES | Keyword::OPTIONS + if parser.peek_token_ref().token == Token::LParen => + { + if seen_options { + return Err(plan_datafusion_err!( + "CREATE TABLE AS SELECT accepts only one table options clause" + )); + } + seen_options = true; } - } else { - value.push(ch); - } - } - if let Some(end) = end { - if trimmed[end..].trim().is_empty() { - return Ok(normalize_schema_identifier( - &Ident::with_quote(first as char, value), - enable_ident_normalization, - )); - } + _ => {} + }, + _ => {} } - return Err(DataFusionError::Plan(format!( - "Invalid quoted identifier in PARTITIONED BY: {trimmed}" - ))); - } - - let parts: Vec<&str> = trimmed.split_whitespace().collect(); - match parts.len() { - 1 => Ok(normalize_schema_identifier( - &Ident::new(parts[0]), - enable_ident_normalization, - )), - _ => Err(DataFusionError::Plan(format!( - "PARTITIONED BY column '{}' should not specify a type. \ - Use column references only, e.g. PARTITIONED BY ({})", - parts[0], parts[0] - ))), } } -fn split_partition_columns(inner: &str) -> DFResult> { - let mut columns = Vec::new(); - let mut start = 0; - let mut quote = None; - let mut chars = inner.char_indices().peekable(); - while let Some((idx, ch)) = chars.next() { - match quote { - Some(q) if ch == q => { - if chars.peek().is_some_and(|(_, next)| *next == q) { - chars.next(); - } else { - quote = None; - } - } - Some(_) => {} - None if ch == '"' || ch == '`' => quote = Some(ch), - None if ch == ',' => { - columns.push(&inner[start..idx]); - start = idx + ch.len_utf8(); - } - None => {} +fn sql_byte_offset(sql: &str, location: Location) -> usize { + let mut offset = 0; + for (index, line) in sql.split_inclusive('\n').enumerate() { + if index + 1 == location.line as usize { + return offset + + line + .char_indices() + .nth(location.column as usize - 1) + .map_or(line.len(), |(index, _)| index); } + offset += line.len(); } - if quote.is_some() { - return Err(DataFusionError::Plan( - "Unterminated quoted identifier in PARTITIONED BY".to_string(), - )); - } - columns.push(&inner[start..]); - Ok(columns) + offset } -/// Extract `PARTITIONED BY (col1, col2, ...)` from SQL before parsing. -/// -/// Paimon only allows column references (no types) in PARTITIONED BY. -/// Since sqlparser's GenericDialect requires types in column definitions, -/// we extract and validate the clause ourselves, then strip it from the SQL -/// so sqlparser can parse the rest. +/// GenericDialect expects typed Hive partition columns. Parse Paimon's list of +/// column references separately and leave the AS query intact. fn extract_partition_by( sql: &str, enable_ident_normalization: bool, ) -> DFResult<(String, Vec)> { - let Some((kw_start, by_end)) = find_partitioned_by(sql) else { - return Ok((sql.to_string(), vec![])); + let Some(mut parser) = create_table_parser(sql) else { + return Ok((sql.to_owned(), vec![])); }; - - let after_by = sql[by_end..].trim_start(); - let paren_start = by_end + (sql[by_end..].len() - after_by.len()); - - if !after_by.starts_with('(') { - return Err(DataFusionError::Plan( - "Expected '(' after PARTITIONED BY".to_string(), - )); - } - - let inner_start = paren_start + 1; - let mut depth = 1; - let mut paren_end = None; - let mut quote = None; - let mut chars = sql[inner_start..].char_indices().peekable(); - while let Some((i, ch)) = chars.next() { - match quote { - Some(q) if ch == q => { - if chars.peek().is_some_and(|(_, next)| *next == q) { - chars.next(); - } else { - quote = None; - } + let mut depth = 0; + let start = loop { + let token = parser.next_token(); + match token.token { + Token::LParen => depth += 1, + Token::RParen => depth -= 1, + Token::Word(word) if depth == 0 && word.keyword == Keyword::AS => { + return Ok((sql.to_owned(), vec![])); } - Some(_) => {} - None if ch == '"' || ch == '`' => quote = Some(ch), - None if ch == '(' => depth += 1, - None if ch == ')' => { - depth -= 1; - if depth == 0 { - paren_end = Some(inner_start + i); - break; - } + Token::Word(word) + if depth == 0 + && word.keyword == Keyword::PARTITIONED + && parser.parse_keyword(Keyword::BY) => + { + break token.span.start; } - None => {} + Token::EOF => return Ok((sql.to_owned(), vec![])), + _ => {} } + }; + if !parser.consume_token(&Token::LParen) { + return Err(plan_datafusion_err!("Expected '(' after PARTITIONED BY")); } - let paren_end = paren_end.ok_or_else(|| { - DataFusionError::Plan("Unmatched '(' in PARTITIONED BY clause".to_string()) - })?; - - let inner = sql[inner_start..paren_end].trim(); - if inner.is_empty() { - return Err(DataFusionError::Plan( - "PARTITIONED BY must specify at least one column".to_string(), + if parser.peek_token_ref().token == Token::RParen { + return Err(plan_datafusion_err!( + "PARTITIONED BY must specify at least one column" )); } - - let mut partition_keys = Vec::new(); - for token in split_partition_columns(inner)? { - partition_keys.push(parse_partition_column(token, enable_ident_normalization)?); + let keys = parser.parse_comma_separated(Parser::parse_identifier)?; + let end = parser.next_token(); + match end.token { + Token::RParen => {} + Token::EOF => return Err(plan_datafusion_err!("Unmatched '(' in PARTITIONED BY clause")), + other => return Err(plan_datafusion_err!( + "Expected ',' or ')' in PARTITIONED BY, found {other}. Partition columns should not specify a type" + )), } - - let clause_end = paren_end + 1; - let mut rewritten = String::with_capacity(sql.len()); - rewritten.push_str(&sql[..kw_start]); - rewritten.push_str(&sql[clause_end..]); - Ok((rewritten, partition_keys)) + let normalizer = IdentNormalizer::new(enable_ident_normalization); + let keys = keys + .into_iter() + .map(|key| normalizer.normalize(key)) + .collect(); + let mut rewritten = sql.to_owned(); + rewritten.replace_range( + sql_byte_offset(sql, start)..sql_byte_offset(sql, end.span.end), + "", + ); + Ok((rewritten, keys)) } pub(crate) fn normalize_schema_identifier( @@ -3985,27 +3793,27 @@ fn object_name_to_single_identifier( } /// Extract key-value pairs from [`CreateTableOptions`]. -fn extract_options(opts: &CreateTableOptions) -> DFResult> { +fn extract_options(opts: &CreateTableOptions) -> DFResult> { let sql_options = match opts { CreateTableOptions::With(options) | CreateTableOptions::Options(options) | CreateTableOptions::TableProperties(options) | CreateTableOptions::Plain(options) => options, - CreateTableOptions::None => return Ok(Vec::new()), + CreateTableOptions::None => return Ok(HashMap::new()), }; sql_options .iter() .filter(|option| !matches!(option, SqlOption::Comment(_))) .map(|opt| match opt { SqlOption::KeyValue { key, value } => { - let v = value.to_string(); - // Strip surrounding quotes from the value if present. - let v = v - .strip_prefix('\'') - .and_then(|s| s.strip_suffix('\'')) - .unwrap_or(&v) - .to_string(); - Ok((key.value.clone(), v)) + let value = match value { + SqlExpr::Value(literal) => literal + .clone() + .into_string() + .unwrap_or_else(|| literal.to_string()), + expr => expr.to_string(), + }; + Ok((key.value.clone(), value)) } other => Err(DataFusionError::Plan(format!( "Unsupported table option: {other}" @@ -4429,284 +4237,125 @@ fn datum_to_constant_array( } } -struct VersionAsOfInfo { - table_name: String, - version: String, - /// Byte range (start, end) covering "table_name VERSION AS OF n" - clause_range: (usize, usize), -} - -struct TimestampAsOfInfo { - table_name: String, - timestamp: String, - /// Byte range (start, end) covering "table_name TIMESTAMP AS OF 'ts'" - clause_range: (usize, usize), +struct TimeTravelSource { + binding: String, + table: ObjectName, + clause: TimeTravelClause, } -/// A parsed `VERSION AS OF` or `TIMESTAMP AS OF` selector. +#[derive(Debug, PartialEq)] enum TimeTravelClause { Version(String), - TimestampMillis(i64), + Timestamp(String), } -/// Locate the table before a time-travel keyword without treating delimiters -/// inside quoted identifiers as boundaries. -fn time_travel_table_before(prefix: &str) -> (usize, &str) { - let prefix = prefix.trim_end(); - let mut quote = None; - for (index, ch) in prefix.char_indices().rev() { - if let Some(delimiter) = quote { - if ch == delimiter { - quote = None; - } - } else if matches!(ch, '"' | '`') { - // Adjacent escaped quotes toggle the quote state twice. - quote = Some(ch); - } else if ch.is_whitespace() || matches!(ch, ',' | '(') { - let start = index + ch.len_utf8(); - return (start, &prefix[start..]); - } +/// Defer snapshot lookups so `IF NOT EXISTS` can skip the source query. +fn mask_time_travel_sources(sql: &mut String) -> Vec { + let lowercase = sql.to_ascii_lowercase(); + if !lowercase.contains("version") && !lowercase.contains("timestamp") { + return Vec::new(); } - (0, prefix) -} - -/// Check for a time-travel keyword outside quoted strings, identifiers, and comments. -fn contains_time_travel_keyword(sql: &str) -> bool { - let bytes = sql.as_bytes(); - let len = bytes.len(); - let mut i = 0; - while i < len { - match bytes[i] { - quote @ (b'\'' | b'"' | b'`') => { - // Skip a quoted string or identifier, including doubled quotes. - i += 1; - while i < len { - if bytes[i] == quote { - i += 1; - if i < len && bytes[i] == quote { - i += 1; // escaped quote - } else { - break; - } - } else { - i += 1; - } - } - } - b'-' if i + 1 < len && bytes[i + 1] == b'-' => { - // Skip line comment - i += 2; - while i < len && bytes[i] != b'\n' { - i += 1; - } + let tokens: Vec<_> = Tokenizer::new(&GenericDialect, sql) + .tokenize_with_location() + .unwrap_or_default() + .into_iter() + .filter(|token| !matches!(token.token, Token::Whitespace(_))) + .collect(); + let mut sources = Vec::new(); + for (index, clause) in tokens.windows(4).enumerate().rev() { + if index == 0 + || !matches!(tokens[index - 1].token, Token::Word(_)) + || !matches!(&clause[1].token, Token::Word(word) if word.keyword == Keyword::AS) + || !matches!(&clause[2].token, Token::Word(word) if word.keyword == Keyword::OF) + { + continue; + } + let Token::Word(word) = &clause[0].token else { + continue; + }; + let selector = match (word.keyword, &clause[3].token) { + (Keyword::VERSION, Token::SingleQuotedString(tag)) => { + TimeTravelClause::Version(tag.clone()) } - b'/' if i + 1 < len && bytes[i + 1] == b'*' => { - // Skip block comment - i += 2; - while i + 1 < len { - if bytes[i] == b'*' && bytes[i + 1] == b'/' { - i += 2; - break; - } - i += 1; - } + (Keyword::VERSION, Token::Number(id, false)) + if id.bytes().all(|byte| byte.is_ascii_digit()) => + { + TimeTravelClause::Version(id.clone()) } - _ => { - // Check for keywords - if i + 14 <= len && bytes[i..i + 14].eq_ignore_ascii_case(b"version as of ") { - return true; - } - if i + 16 <= len && bytes[i..i + 16].eq_ignore_ascii_case(b"timestamp as of ") { - return true; - } - i += 1; + (Keyword::TIMESTAMP, Token::SingleQuotedString(timestamp)) => { + TimeTravelClause::Timestamp(timestamp.clone()) } + _ => continue, + }; + let mut start = index - 1; + while start >= 2 + && matches!(tokens[start - 1].token, Token::Period) + && matches!(tokens[start - 2].token, Token::Word(_)) + { + start -= 2; } + let table_start = sql_byte_offset(sql, tokens[start].span.start); + let identifiers: Vec<_> = tokens[start..index] + .iter() + .filter_map(|token| match &token.token { + Token::Word(word) => Some(word.to_ident(token.span)), + _ => None, + }) + .collect(); + let end = sql_byte_offset(sql, clause[3].span.end); + let binding = format!("__paimon_tt_{}", uuid::Uuid::new_v4().as_simple()); + let separator = if start > 0 && tokens[start - 1].span.end == tokens[start].span.start { + " " + } else { + "" + }; + sql.replace_range(table_start..end, &format!("{separator}TABLE({binding})")); + sources.push(TimeTravelSource { + binding, + table: ObjectName::from(identifiers), + clause: selector, + }); } - false + sources.reverse(); + sources } -/// Extract **all** `VERSION AS OF ` or `VERSION AS OF ''` clauses from a -/// SQL string, skipping quoted strings, identifiers, and comments. -fn extract_all_version_as_of(sql: &str) -> Vec { - let bytes = sql.as_bytes(); - let len = bytes.len(); - let mut i = 0; - let mut results = Vec::new(); - - while i < len { - match bytes[i] { - quote @ (b'\'' | b'"' | b'`') => { - // Skip a quoted string or identifier, including doubled quotes. - i += 1; - while i < len { - if bytes[i] == quote { - i += 1; - if i < len && bytes[i] == quote { - i += 1; // escaped quote - } else { - break; - } - } else { - i += 1; - } - } - } - b'-' if i + 1 < len && bytes[i + 1] == b'-' => { - // Skip line comment - i += 2; - while i < len && bytes[i] != b'\n' { - i += 1; - } - } - b'/' if i + 1 < len && bytes[i + 1] == b'*' => { - // Skip block comment - i += 2; - while i + 1 < len { - if bytes[i] == b'*' && bytes[i + 1] == b'/' { - i += 2; - break; - } - i += 1; - } - } - _ => { - if i + 14 <= len && bytes[i..i + 14].eq_ignore_ascii_case(b"version as of ") { - let kw_start = i; - let val_start = i + 14; - let remaining = &sql[val_start..]; - - // Parse either a quoted tag name or a numeric snapshot ID - let version = if let Some(after_quote) = remaining.strip_prefix('\'') { - // Tag name: VERSION AS OF 'tagname' - if let Some(close_quote) = after_quote.find('\'') { - after_quote[..close_quote].to_string() - } else { - i += 1; - continue; - } - } else { - // Numeric snapshot ID: VERSION AS OF 1 - let v: String = remaining - .chars() - .take_while(|c| c.is_ascii_digit()) - .collect(); - if v.is_empty() { - i += 1; - continue; - } - v - }; - - let is_quoted = remaining.starts_with('\''); - let val_end = if is_quoted { - val_start + version.len() + 2 // 2 quotes - } else { - val_start + version.len() - }; - - let (table_start, table_name) = time_travel_table_before(&sql[..kw_start]); - - if !table_name.is_empty() { - results.push(VersionAsOfInfo { - table_name: table_name.to_string(), - version, - clause_range: (table_start, val_end), - }); - } - - i = val_end; - } else { - i += 1; - } +fn validate_time_travel_sources( + statement: &Statement, + sources: &[TimeTravelSource], +) -> DFResult<()> { + use datafusion::sql::sqlparser::ast::{Visit, Visitor}; + use std::ops::ControlFlow; + + struct Bindings<'a>(HashSet<&'a str>); + impl Visitor for Bindings<'_> { + type Break = (); + fn pre_visit_table_factor(&mut self, factor: &TableFactor) -> ControlFlow<()> { + if let TableFactor::TableFunction { + expr: SqlExpr::Identifier(binding), + .. + } = factor + { + self.0.remove(binding.value.as_str()); } + ControlFlow::Continue(()) } } - - results -} - -/// Extract all time-travel timestamp clauses outside quoted strings, -/// identifiers, and comments. -fn extract_all_timestamp_as_of(sql: &str) -> Vec { - let bytes = sql.as_bytes(); - let len = bytes.len(); - let mut i = 0; - let mut results = Vec::new(); - - while i < len { - match bytes[i] { - quote @ (b'\'' | b'"' | b'`') => { - // Skip a quoted string or identifier, including doubled quotes. - i += 1; - while i < len { - if bytes[i] == quote { - i += 1; - if i < len && bytes[i] == quote { - i += 1; // escaped quote - } else { - break; - } - } else { - i += 1; - } - } - } - b'-' if i + 1 < len && bytes[i + 1] == b'-' => { - // Skip line comment - i += 2; - while i < len && bytes[i] != b'\n' { - i += 1; - } - } - b'/' if i + 1 < len && bytes[i + 1] == b'*' => { - // Skip block comment - i += 2; - while i + 1 < len { - if bytes[i] == b'*' && bytes[i + 1] == b'/' { - i += 2; - break; - } - i += 1; - } - } - _ => { - if i + 16 <= len && bytes[i..i + 16].eq_ignore_ascii_case(b"timestamp as of ") { - let kw_start = i; - let val_start = i + 16; - let remaining = &sql[val_start..]; - - // Read the quoted timestamp string - if !remaining.starts_with('\'') { - i += 1; - continue; - } - if let Some(close_quote) = remaining[1..].find('\'') { - let timestamp = remaining[1..close_quote + 1].to_string(); - let val_end = val_start + close_quote + 2; // skip both quotes - - let (table_start, table_name) = time_travel_table_before(&sql[..kw_start]); - - if !table_name.is_empty() { - results.push(TimestampAsOfInfo { - table_name: table_name.to_string(), - timestamp, - clause_range: (table_start, val_end), - }); - } - - i = val_end; - } else { - i += 1; - } - } else { - i += 1; - } - } + let mut bindings = Bindings(sources.iter().map(|s| s.binding.as_str()).collect()); + match statement { + Statement::CreateTable(ct) => { + let _ = ct.query.visit(&mut bindings); + } + _ => { + let _ = statement.visit(&mut bindings); } } - - results + if !bindings.0.is_empty() { + return Err(DataFusionError::Plan( + "Time travel requires a source table reference".into(), + )); + } + Ok(()) } /// Return an empty DataFrame with a single "result" column containing "OK". @@ -8723,15 +8372,13 @@ mod tests { #[test] fn test_extract_options_with_kv() { - // Parse a CREATE TABLE with WITH options to get a real CreateTableOptions let dialect = GenericDialect {}; let stmts = Parser::parse_sql(&dialect, "CREATE TABLE t (id INT) WITH ('bucket' = '4')").unwrap(); if let Statement::CreateTable(ct) = &stmts[0] { let opts = extract_options(&ct.table_options).unwrap(); assert_eq!(opts.len(), 1); - assert_eq!(opts[0].0, "bucket"); - assert_eq!(opts[0].1, "4"); + assert_eq!(opts["bucket"], "4"); } else { panic!("expected CreateTable"); } @@ -8895,6 +8542,40 @@ mod tests { } } + #[tokio::test] + async fn test_ctas_existing_target_without_catalog_listing() { + let catalog = Arc::new(MockCatalog::new()); + let sql_context = make_sql_context(catalog.clone()).await; + for table_type in ["table", "iceberg-table"] { + let schema = PaimonSchema::builder() + .column("id", PaimonDataType::Int(IntType::new())) + .option("type", table_type) + .build() + .unwrap(); + *catalog.existing_table.lock().unwrap() = Some(Table::new( + paimon::io::FileIOBuilder::new("memory").build().unwrap(), + Identifier::new("mydb", "t1"), + "memory:/warehouse/mydb.db/t1".to_string(), + paimon::spec::TableSchema::new(0, &schema), + None, + )); + assert!(catalog.list_tables("mydb").await.unwrap().is_empty()); + sql_context + .sql("CREATE TABLE IF NOT EXISTS mydb.t1 AS SELECT missing_function(id) FROM missing_source") + .await + .unwrap() + .collect() + .await + .unwrap(); + let error = sql_context + .sql("CREATE TABLE mydb.t1 AS SELECT missing_function(id) FROM missing_source") + .await + .unwrap_err(); + assert!(error.to_string().contains("already exists"), "{error}"); + assert!(catalog.take_calls().is_empty()); + } + } + #[tokio::test] async fn test_create_table_with_options() { let catalog = Arc::new(MockCatalog::new()); @@ -9587,50 +9268,34 @@ mod tests { #[test] fn test_extract_partition_by_skips_string_literal() { - let sql = - "CREATE TABLE t (id INT) WITH ('note' = 'PARTITIONED BY (x)') PARTITIONED BY (id)"; - let (rewritten, keys) = extract_partition_by(sql, true).unwrap(); - assert_eq!(keys, vec!["id"]); - assert!(rewritten.contains("WITH")); - assert!(rewritten.contains("'PARTITIONED BY (x)'")); - } - - #[test] - fn test_extract_partition_by_skips_line_comment() { - let sql = "CREATE TABLE t (id INT) -- PARTITIONED BY (x)\nPARTITIONED BY (id)"; - let (_, keys) = extract_partition_by(sql, true).unwrap(); - assert_eq!(keys, vec!["id"]); - } - - #[test] - fn test_extract_partition_by_double_quoted_identifier() { - let (_, keys) = extract_partition_by( - "CREATE TABLE t (\"order\" INT) PARTITIONED BY (\"order\")", - true, - ) - .unwrap(); - assert_eq!(keys, vec!["order"]); - } - - #[test] - fn test_extract_partition_by_double_quoted_identifier_with_escaped_quote_and_comma() { - let (_, keys) = extract_partition_by( - "CREATE TABLE t (\"a\"\"b,c\" INT, `d``e,f` INT) \ - PARTITIONED BY (\"a\"\"b,c\", `d``e,f`)", - true, - ) - .unwrap(); - assert_eq!(keys, vec!["a\"b,c", "d`e,f"]); + for literal in [ + "'PARTITIONED BY (x)'", + "$$PARTITIONED BY (x)$$", + "$tag$PARTITIONED BY (x)$tag$", + ] { + let sql = + format!("CREATE TABLE t (id INT) WITH ('note' = {literal}) PARTITIONED BY (id)"); + let (rewritten, keys) = extract_partition_by(&sql, true).unwrap(); + assert_eq!(keys, vec!["id"]); + assert!(rewritten.contains("WITH")); + assert!(rewritten.contains(literal)); + } } #[test] - fn test_extract_partition_by_backtick_quoted_identifier() { - let (_, keys) = extract_partition_by( - "CREATE TABLE t (`order` INT) PARTITIONED BY (`order`)", - true, - ) - .unwrap(); - assert_eq!(keys, vec!["order"]); + fn test_extract_partition_by_quoted_identifiers_and_comments() { + for (clause, expected) in [ + (r#"("order")"#, vec!["order"]), + ("(`order`)", vec!["order"]), + ( + "/* layout */ (\"a\"\"b,c\" /* ,) */, -- second key\n `d``e,f`)", + vec!["a\"b,c", "d`e,f"], + ), + ] { + let sql = format!("CREATE TABLE t PARTITIONED BY {clause} AS SELECT 1"); + let (_, keys) = extract_partition_by(&sql, true).unwrap(); + assert_eq!(keys, expected, "{clause}"); + } } #[test] @@ -9649,34 +9314,57 @@ mod tests { } #[test] - fn test_extract_partition_by_skips_block_comment() { - let sql = "CREATE TABLE t (id INT) /* PARTITIONED BY (x) */ PARTITIONED BY (id)"; - let (rewritten, keys) = extract_partition_by(sql, true).unwrap(); - assert_eq!(keys, vec!["id"]); - assert!(rewritten.contains("/* PARTITIONED BY (x) */")); + fn test_extract_partition_by_skips_comments() { + for comment in [ + "-- PARTITIONED BY (x)", + "/* PARTITIONED BY (x) */", + "/* outer /* inner */ PARTITIONED BY (x) */", + ] { + let sql = + format!("CREATE TABLE \"中文\" (id INT) {comment}\nPARTITIONED /* key */ BY (id)"); + let (rewritten, keys) = extract_partition_by(&sql, true).unwrap(); + assert_eq!(keys, vec!["id"]); + assert!(rewritten.contains(comment)); + assert!(rewritten.starts_with("CREATE TABLE \"中文\" (id INT)")); + } } #[test] - fn test_looks_like_create_table() { - assert!(looks_like_create_table("CREATE TABLE t (id INT)")); - assert!(looks_like_create_table(" create table t (id INT)")); - assert!(looks_like_create_table( + fn test_create_table_parser() { + for sql in [ + "CREATE TABLE t (id INT)", + " create table t (id INT)", "CREATE TABLE IF NOT EXISTS t (id INT)", - )); - assert!(looks_like_create_table( "/* note */ CREATE TABLE t (id INT)", - )); - assert!(looks_like_create_table( "-- comment\nCREATE TABLE t (id INT)", - )); - assert!(looks_like_create_table( "/* a */ /* b */ CREATE TABLE t (id INT)", - )); - assert!(!looks_like_create_table("ALTER TABLE t ADD COLUMN x INT")); - assert!(!looks_like_create_table("SELECT 1")); - assert!(!looks_like_create_table( + "CREATE /* note */ TABLE t AS SELECT 1", + "CREATE TEMP TABLE t AS SELECT 1", + "CREATE TEMPORARY TABLE t AS SELECT 1", + "CREATE OR REPLACE TABLE t AS SELECT * FROM s VERSION AS OF 1", + "CREATE OR ALTER TABLE t AS SELECT * FROM s VERSION AS OF 1", + "CREATE EXTERNAL TABLE t AS SELECT 1", + "CREATE GLOBAL TABLE t AS SELECT 1", + "CREATE TRANSIENT TABLE t AS SELECT 1", + "; CREATE TABLE t AS SELECT 1", + "/* start */ ; ; -- empty statements\n; CREATE TABLE t AS SELECT 1", + "; CREATE OR REPLACE TABLE t AS SELECT * FROM s VERSION AS OF 1", + ] { + assert!(create_table_parser(sql).is_some(), "{sql}"); + } + for sql in [ + "ALTER TABLE t ADD COLUMN x INT", + "SELECT 1", "SELECT aaaaaaaaaaaaaaaaaaaa中文 FROM t", - )); + "CREATE VIEW v AS SELECT 1", + "CREATE TEMP VIEW v AS SELECT 1", + "CREATE TABLE_SUFFIX t (id INT)", + "CREATE \"TABLE\" t (id INT)", + "; ; -- only empty statements\n;", + "; SELECT 1; CREATE TABLE t AS SELECT 1", + ] { + assert!(create_table_parser(sql).is_none(), "{sql}"); + } } // ==================== partition key validation tests ==================== @@ -10700,212 +10388,81 @@ mod tests { } #[test] - fn test_extract_version_as_of() { - let sql = "SELECT id, name FROM paimon.default.time_travel_table VERSION AS OF 1"; - let infos = extract_all_version_as_of(sql); - assert_eq!(infos.len(), 1); - let info = &infos[0]; - assert_eq!(info.version, "1"); - assert_eq!(info.table_name, "paimon.default.time_travel_table"); - let rewritten = format!( - "{}__uuid{}", - &sql[..info.clause_range.0], - &sql[info.clause_range.1..] - ); - assert_eq!(rewritten, "SELECT id, name FROM __uuid"); - } - - #[test] - fn test_extract_version_as_of_multi_digit() { - let sql = "SELECT * FROM mydb.t VERSION AS OF 42"; - let infos = extract_all_version_as_of(sql); - assert_eq!(infos.len(), 1); - let info = &infos[0]; - assert_eq!(info.version, "42"); - assert_eq!(info.table_name, "mydb.t"); - let rewritten = format!( - "{}__uuid{}", - &sql[..info.clause_range.0], - &sql[info.clause_range.1..] - ); - assert_eq!(rewritten, "SELECT * FROM __uuid"); - } - - #[test] - fn test_extract_version_as_of_case_insensitive() { - let sql = "SELECT * FROM t version as of 5"; - let infos = extract_all_version_as_of(sql); - assert_eq!(infos.len(), 1); - let info = &infos[0]; - assert_eq!(info.version, "5"); - assert_eq!(info.table_name, "t"); - let rewritten = format!( - "{}__uuid{}", - &sql[..info.clause_range.0], - &sql[info.clause_range.1..] - ); - assert_eq!(rewritten, "SELECT * FROM __uuid"); - } - - #[test] - fn test_extract_version_as_of_not_present() { - let sql = "SELECT * FROM t"; - assert!(extract_all_version_as_of(sql).is_empty()); - } - - #[test] - fn test_extract_version_as_of_tag() { - let sql = "SELECT id, name FROM paimon.default.t VERSION AS OF 'snapshot1'"; - let infos = extract_all_version_as_of(sql); - assert_eq!(infos.len(), 1); - let info = &infos[0]; - assert_eq!(info.version, "snapshot1"); - assert_eq!(info.table_name, "paimon.default.t"); - let rewritten = format!( - "{}__uuid{}", - &sql[..info.clause_range.0], - &sql[info.clause_range.1..] - ); - assert_eq!(rewritten, "SELECT id, name FROM __uuid"); - } - - #[test] - fn test_extract_version_as_of_tag_case_insensitive() { - let sql = "SELECT * FROM t version as of 'my_tag'"; - let infos = extract_all_version_as_of(sql); - assert_eq!(infos.len(), 1); - let info = &infos[0]; - assert_eq!(info.version, "my_tag"); - assert_eq!(info.table_name, "t"); - let rewritten = format!( - "{}__uuid{}", - &sql[..info.clause_range.0], - &sql[info.clause_range.1..] - ); - assert_eq!(rewritten, "SELECT * FROM __uuid"); - } - - #[test] - fn test_extract_version_as_of_numeric_still_works() { - let sql = "SELECT * FROM t VERSION AS OF 123"; - let infos = extract_all_version_as_of(sql); - assert_eq!(infos.len(), 1); - assert_eq!(infos[0].version, "123"); - assert_eq!(infos[0].table_name, "t"); - } - - #[test] - fn test_extract_version_as_of_multiple() { - // JOIN two time-travel tables - let sql = "SELECT * FROM t1 VERSION AS OF 1 JOIN t2 VERSION AS OF 2 ON t1.id = t2.id"; - let infos = extract_all_version_as_of(sql); - assert_eq!(infos.len(), 2); - assert_eq!(infos[0].version, "1"); - assert_eq!(infos[0].table_name, "t1"); - assert_eq!(infos[1].version, "2"); - assert_eq!(infos[1].table_name, "t2"); - } - - #[test] - fn test_extract_version_as_of_skips_string_literal() { - let sql = "SELECT * FROM t WHERE note = 'version as of 1'"; - let infos = extract_all_version_as_of(sql); - assert!(infos.is_empty()); - } - - #[test] - fn test_extract_version_as_of_skips_comment() { - let sql = "SELECT * FROM t -- version as of 1\n WHERE id > 0"; - let infos = extract_all_version_as_of(sql); - assert!(infos.is_empty()); + fn test_mask_time_travel_sources() { + use TimeTravelClause::{Timestamp, Version}; + + for (selector, expected) in [ + ("VERSION AS OF 1", Version("1".into())), + ("version as of 123", Version("123".into())), + ("VERSION AS OF 'my_tag'", Version("my_tag".into())), + ("VERSION AS OF 'tag''quoted'", Version("tag'quoted".into())), + ( + "/* table */ VERSION\nAS /* selector */ OF 42", + Version("42".into()), + ), + ( + "timestamp as of '2024-01-15 10:30:00'", + Timestamp("2024-01-15 10:30:00".into()), + ), + ( + "/* table */ TIMESTAMP\nAS /* selector */ OF '2024-01-15'", + Timestamp("2024-01-15".into()), + ), + ] { + let mut sql = format!( + r#"SELECT 'İ' FROM paimon."my db"."with ""quote"", (space" {selector} AS s"# + ); + let sources = mask_time_travel_sources(&mut sql); + assert_eq!(sources.len(), 1, "{selector}"); + assert_eq!(sources[0].clause, expected); + assert_eq!( + sources[0].table.to_string(), + r#"paimon."my db"."with ""quote"", (space""# + ); + assert_eq!( + sql, + format!("SELECT 'İ' FROM TABLE({}) AS s", sources[0].binding) + ); + } } #[test] - fn test_time_travel_keeps_quoted_table_reference() { - let table = r#"paimon."my db"."with ""quote"", (space""#; - let version_sql = format!("SELECT 'İ' FROM {table} VERSION AS OF 1"); - let versions = extract_all_version_as_of(&version_sql); - assert_eq!(versions.len(), 1); - assert_eq!(versions[0].table_name, table); - assert_eq!( - &version_sql[versions[0].clause_range.0..versions[0].clause_range.1], - format!("{table} VERSION AS OF 1") - ); - - let timestamp_sql = format!("SELECT 'İ' FROM {table} TIMESTAMP AS OF '2024-01-01'"); - let timestamps = extract_all_timestamp_as_of(×tamp_sql); - assert_eq!(timestamps.len(), 1); - assert_eq!(timestamps[0].table_name, table); + fn test_mask_multiple_time_travel_sources() { + let mut sql = + "SELECT * FROM t1 VERSION AS OF 1 JOIN t2 VERSION AS OF 2 ON t1.id = t2.id".to_string(); + let sources = mask_time_travel_sources(&mut sql); + assert_eq!(sources.len(), 2); + assert_eq!(sources[0].table.to_string(), "t1"); + assert_eq!(sources[0].clause, TimeTravelClause::Version("1".into())); + assert_eq!(sources[1].table.to_string(), "t2"); + assert_eq!(sources[1].clause, TimeTravelClause::Version("2".into())); assert_eq!( - ×tamp_sql[timestamps[0].clause_range.0..timestamps[0].clause_range.1], - format!("{table} TIMESTAMP AS OF '2024-01-01'") - ); - } - - #[test] - fn test_contains_time_travel_keyword() { - assert!(contains_time_travel_keyword( - "SELECT * FROM t VERSION AS OF 1" - )); - assert!(contains_time_travel_keyword( - "SELECT * FROM t TIMESTAMP AS OF '2024-01-01 00:00:00'" - )); - // Inside string literal — should NOT match - assert!(!contains_time_travel_keyword( - "SELECT * FROM t WHERE note = 'version as of 1'" - )); - // Inside comment — should NOT match - assert!(!contains_time_travel_keyword( - "SELECT * FROM t -- version as of 1" - )); - assert!(!contains_time_travel_keyword( - "SELECT * FROM t /* timestamp as of now */ WHERE id > 0" - )); - assert!(!contains_time_travel_keyword( - r#"SELECT * FROM "version as of 1""# - )); - assert!(!contains_time_travel_keyword( - "SELECT * FROM `timestamp as of now`" - )); - // No keyword at all - assert!(!contains_time_travel_keyword("SELECT * FROM t")); - } - - #[test] - fn test_extract_timestamp_as_of() { - let sql = "SELECT * FROM paimon.default.t TIMESTAMP AS OF '2024-01-15 10:30:00'"; - let infos = extract_all_timestamp_as_of(sql); - assert_eq!(infos.len(), 1); - let info = &infos[0]; - assert_eq!(info.timestamp, "2024-01-15 10:30:00"); - assert_eq!(info.table_name, "paimon.default.t"); - let rewritten = format!( - "{}__uuid{}", - &sql[..info.clause_range.0], - &sql[info.clause_range.1..] - ); - assert_eq!(rewritten, "SELECT * FROM __uuid"); - } - - #[test] - fn test_extract_timestamp_as_of_case_insensitive() { - let sql = "SELECT * FROM t timestamp as of '2024-06-01 00:00:00'"; - let infos = extract_all_timestamp_as_of(sql); - assert_eq!(infos.len(), 1); - let info = &infos[0]; - assert_eq!(info.timestamp, "2024-06-01 00:00:00"); - assert_eq!(info.table_name, "t"); - let rewritten = format!( - "{}__uuid{}", - &sql[..info.clause_range.0], - &sql[info.clause_range.1..] + sql, + format!( + "SELECT * FROM TABLE({}) JOIN TABLE({}) ON t1.id = t2.id", + sources[0].binding, sources[1].binding + ) ); - assert_eq!(rewritten, "SELECT * FROM __uuid"); } #[test] - fn test_extract_timestamp_as_of_not_present() { - let sql = "SELECT * FROM t"; - assert!(extract_all_timestamp_as_of(sql).is_empty()); + fn test_time_travel_ignores_literals_comments_and_aliases() { + for sql in [ + "SELECT * FROM t", + "SELECT * FROM t WHERE note = 'version as of 1'", + "SELECT $$source VERSION AS OF 1$$ FROM t", + "SELECT $tag$source TIMESTAMP AS OF '2024-01-01'$tag$ FROM t", + "SELECT * FROM t -- version as of 1", + "SELECT * FROM t /* timestamp as of now */ WHERE id > 0", + "SELECT * FROM t /* outer /* inner */ source VERSION AS OF 1 */", + r#"SELECT * FROM "version as of 1""#, + "SELECT * FROM `timestamp as of now`", + "SELECT conversion AS of_value FROM t", + "SELECT version AS of FROM t", + ] { + let mut masked = sql.to_string(); + assert!(mask_time_travel_sources(&mut masked).is_empty(), "{sql}"); + assert_eq!(masked, sql); + } } } diff --git a/crates/integrations/datafusion/src/sql_context/ctas.rs b/crates/integrations/datafusion/src/sql_context/ctas.rs new file mode 100644 index 000000000..993404e05 --- /dev/null +++ b/crates/integrations/datafusion/src/sql_context/ctas.rs @@ -0,0 +1,285 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Persistent CREATE TABLE AS SELECT. Population failures retain the table +//! because concurrent writers can use it as soon as creation succeeds. + +use std::ops::ControlFlow; +use std::sync::Arc; + +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::error::{DataFusionError, Result as DFResult}; +use datafusion::execution::context::SQLOptions; +use datafusion::execution::session_state::SessionState; +use datafusion::execution::TaskContext; +use datafusion::physical_plan::{execute_stream, ExecutionPlan}; +use datafusion::prelude::DataFrame; +use datafusion::sql::planner::object_name_to_table_reference; +use datafusion::sql::sqlparser::ast::helpers::stmt_create_table::CreateTableBuilder; +use datafusion::sql::sqlparser::ast::{ + CreateTable, CreateTableOptions, Query, Select, SqlOption, Statement, Visit, Visitor, +}; +use futures::StreamExt; +use paimon::spec::{CoreOptions, DataType as PaimonDataType, TableType}; +use paimon::table::Table; + +use super::{create_table_schema, extract_options, ok_result, SQLContext, TimeTravelSource}; +use crate::error::to_datafusion_error; +use crate::physical_plan::cast_batch_to_schema; + +impl SQLContext { + pub(super) async fn handle_create_table_as( + &self, + ct: &CreateTable, + query: Box, + partition_keys: Vec, + state: SessionState, + options: SQLOptions, + time_travel_sources: &[TimeTravelSource], + ) -> DFResult { + validate_create_table_as(ct, &query)?; + Self::ensure_main_branch_base_table(&ct.name, "CREATE TABLE AS SELECT")?; + let enable_ident_normalization = + state.config_options().sql_parser.enable_ident_normalization; + let target = object_name_to_table_reference(ct.name.clone(), enable_ident_normalization)?; + let (catalog, catalog_name, identifier) = + self.resolve_table_reference(&target, &state.config_options().catalog)?; + + match catalog.check_table_exists(&identifier).await { + Ok(()) if ct.if_not_exists => return ok_result(&self.ctx), + Ok(()) => { + return Err(to_datafusion_error(paimon::Error::TableAlreadyExist { + full_name: identifier.full_name(), + })); + } + Err(paimon::Error::TableNotExist { .. }) => {} + Err(error) => return Err(to_datafusion_error(error)), + } + + let table_options = extract_options(&ct.table_options)?; + let table_type = CoreOptions::new(&table_options) + .table_type() + .map_err(to_datafusion_error)?; + if table_type != TableType::Table { + return Err(DataFusionError::NotImplemented(format!( + "CREATE TABLE AS SELECT only supports native Paimon tables, got '{table_type}'" + ))); + } + + let df = + Box::pin(self.plan_query(state, Statement::Query(query), options, time_travel_sources)) + .await?; + let (state, source) = df.into_parts(); + let task_ctx = state.task_ctx(); + // Optimization can narrow nullability beyond the query's declared schema. + let analyzed = + state + .analyzer() + .execute_and_check(source, state.config_options(), |_, _| {})?; + let schema = create_table_schema( + ct, + partition_keys, + enable_ident_normalization, + Some(analyzed.schema().as_arrow()), + table_options, + )?; + if schema.fields().is_empty() { + return Err(DataFusionError::Plan( + "CREATE TABLE AS SELECT requires at least one output column".into(), + )); + } + for field in schema.fields() { + validate_time_precision(field.data_type())?; + } + let write_schema = paimon::arrow::build_target_arrow_schema(schema.fields()) + .map_err(to_datafusion_error)?; + let optimized = state.optimizer().optimize(analyzed, &state, |_, _| {})?; + let plan = state + .query_planner() + .create_physical_plan(&optimized, &state) + .await?; + + match catalog.create_table_exclusive(&identifier, schema).await { + Ok(()) => {} + Err(paimon::Error::TableAlreadyExist { .. }) if ct.if_not_exists => { + return ok_result(&self.ctx); + } + Err(error) => return Err(to_datafusion_error(error)), + } + let name = format!("{catalog_name}.{}", identifier.full_name()); + let table = catalog.get_table(&identifier).await.map_err(|error| { + DataFusionError::Context( + format!("Failed to load newly created table '{name}' for CREATE TABLE AS SELECT"), + Box::new(to_datafusion_error(error)), + ) + })?; + let (mut response, outcome) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + let result = populate_table(table, plan, task_ctx, write_schema, &mut response) + .await + .map_err(|error| { + DataFusionError::Context( + format!( + "CREATE TABLE AS SELECT failed for '{name}'. \ + Inspect the target table before retrying." + ), + Box::new(error), + ) + }); + if let Err(Err(error)) = response.send(result) { + log::warn!("Cancelled CREATE TABLE AS SELECT for '{name}': {error}"); + } + }); + outcome.await.map_err(|error| { + DataFusionError::Execution(format!("CREATE TABLE AS SELECT worker stopped: {error}")) + })??; + ok_result(&self.ctx) + } +} + +async fn populate_table( + table: Table, + plan: Arc, + task_ctx: Arc, + write_schema: SchemaRef, + response: &mut tokio::sync::oneshot::Sender>, +) -> DFResult<()> { + let resources = crate::memory::writer_resources(&task_ctx).map_err(to_datafusion_error)?; + let builder = table.new_write_builder().with_resources(resources); + let commit = builder.try_new_commit().map_err(to_datafusion_error)?; + let mut writer = builder.new_write().map_err(to_datafusion_error)?; + let prepared = async { + let mut input = execute_stream(plan, task_ctx)?; + loop { + let next = tokio::select! { + biased; + _ = response.closed() => return Ok(None), + batch = input.next() => batch, + }; + let Some(batch) = next else { + break; + }; + let batch = cast_batch_to_schema(batch?, &write_schema)?; + writer + .write_arrow_batch(&batch) + .await + .map_err(to_datafusion_error)?; + } + writer + .prepare_commit() + .await + .map(Some) + .map_err(to_datafusion_error) + } + .await; + writer.close().await; + let messages = prepared.map_err(|error| { + DataFusionError::Context("no commit was attempted".into(), Box::new(error)) + })?; + let Some(messages) = messages else { + return Ok(()); + }; + if response.is_closed() { + commit.abort(&messages).await.map_err(to_datafusion_error)?; + return Ok(()); + } + // A lost response can follow successful publication, so keep the files on commit errors. + Box::pin(commit.commit(messages)).await.map_err(|error| { + DataFusionError::Context( + "publication may have succeeded".into(), + Box::new(to_datafusion_error(error)), + ) + }) +} + +fn validate_time_precision(data_type: &PaimonDataType) -> DFResult<()> { + match data_type { + PaimonDataType::Time(time) if time.precision() > 3 => { + Err(DataFusionError::NotImplemented(format!( + "CREATE TABLE AS SELECT cannot preserve TIME({}); Paimon stores TIME with millisecond precision", + time.precision() + ))) + } + PaimonDataType::Array(array) => validate_time_precision(array.element_type()), + PaimonDataType::Vector(vector) => validate_time_precision(vector.element_type()), + PaimonDataType::Multiset(multiset) => validate_time_precision(multiset.element_type()), + PaimonDataType::Map(map) => { + validate_time_precision(map.key_type())?; + validate_time_precision(map.value_type()) + } + PaimonDataType::Row(row) => row + .fields() + .iter() + .try_for_each(|field| validate_time_precision(field.data_type())), + _ => Ok(()), + } +} + +fn validate_create_table_as(ct: &CreateTable, query: &Query) -> DFResult<()> { + struct ReadOnlyQuery; + impl Visitor for ReadOnlyQuery { + type Break = (); + + fn pre_visit_statement(&mut self, statement: &Statement) -> ControlFlow<()> { + if matches!(statement, Statement::Query(_)) { + ControlFlow::Continue(()) + } else { + ControlFlow::Break(()) + } + } + + fn pre_visit_select(&mut self, select: &Select) -> ControlFlow<()> { + if select.into.is_some() { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + } + } + if let ControlFlow::Break(()) = query.visit(&mut ReadOnlyQuery) { + return Err(DataFusionError::NotImplemented( + "CREATE TABLE AS SELECT requires a read-only query".into(), + )); + } + if !ct.columns.is_empty() { + return Err(DataFusionError::NotImplemented( + "CREATE TABLE AS SELECT with explicit column declarations is not supported".into(), + )); + } + if let CreateTableOptions::Plain(options) = &ct.table_options { + if let Some(option) = options + .iter() + .find(|option| !matches!(option, SqlOption::Comment(_))) + { + return Err(DataFusionError::NotImplemented(format!( + "Unsupported CREATE TABLE AS SELECT option: {option}" + ))); + } + } + // Reject unhandled AST fields, including modifiers added by parser upgrades. + let mut supported = CreateTableBuilder::new(ct.name.clone()) + .if_not_exists(ct.if_not_exists) + .table_options(ct.table_options.clone()) + .build(); + supported.comment = ct.comment.clone(); + if ct != &supported { + return Err(DataFusionError::NotImplemented( + "Unsupported CREATE TABLE AS SELECT modifiers; only IF NOT EXISTS, PARTITIONED BY, table options and COMMENT are supported".into(), + )); + } + Ok(()) +} diff --git a/crates/integrations/datafusion/tests/create_table_as.rs b/crates/integrations/datafusion/tests/create_table_as.rs new file mode 100644 index 000000000..27121f21c --- /dev/null +++ b/crates/integrations/datafusion/tests/create_table_as.rs @@ -0,0 +1,946 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod common; +#[path = "create_table_as/execution.rs"] +mod execution; +#[path = "../../../paimon/tests/mock_server.rs"] +mod mock_server; + +use std::collections::HashMap; +use std::sync::Arc; + +use arrow_array::{Array, Int32Array, Int64Array, RecordBatch}; +use arrow_schema::{DataType as ArrowType, Field, Schema as ArrowSchema}; +use common::{assert_sql_error, create_sql_context, create_test_env, exec, row_count}; +use datafusion::datasource::MemTable; +use paimon::catalog::{Catalog, Identifier, RESTCatalog}; +use paimon::spec::{BigIntType, DataType, IntType, Schema, TableType}; +use paimon::{CatalogOptions, Options}; +use paimon_datafusion::SQLContext; + +async fn query_ids(ctx: &SQLContext, sql: &str) -> Vec { + let batches = ctx.sql(sql).await.unwrap().collect().await.unwrap(); + let mut ids = batches + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .iter() + }) + .map(Option::unwrap) + .collect::>(); + ids.sort_unstable(); + ids +} + +#[tokio::test] +async fn inferred_schema_options_partitioning_and_persistence() { + let (_tmp, catalog) = create_test_env(); + let ctx = create_sql_context(catalog.clone()).await; + let source = Arc::new(ArrowSchema::new(vec![ + Field::new("id", ArrowType::Int32, false), + Field::new("part", ArrowType::Utf8, true), + ])); + let batch = RecordBatch::try_new( + source.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new(arrow_array::StringArray::from(vec![ + Some("a"), + None, + Some("b"), + ])), + ], + ) + .unwrap(); + let partitions = vec![vec![batch.slice(0, 2)], vec![batch.slice(2, 1)]]; + ctx.register_temp_table( + "source", + Arc::new(MemTable::try_new(source, partitions).unwrap()), + ) + .unwrap(); + exec( + &ctx, + "/* start */ ; ; -- empty statements + ; CREATE TABLE result PARTITIONED BY (part) + WITH ('bucket' = '-1', 'file.format' = 'parquet', 'partition.default-name' = 'it''s null') + AS SELECT id + 10 AS id, part FROM source WHERE id > 1", + ) + .await; + let table = catalog + .get_table(&Identifier::new("default", "result")) + .await + .unwrap(); + let schema = table.schema(); + assert_eq!( + schema.fields()[0].data_type(), + &DataType::BigInt(BigIntType::with_nullable(false)) + ); + assert!(schema.fields()[1].data_type().is_nullable()); + assert_eq!(schema.partition_keys(), &["part"]); + assert_eq!( + schema.options(), + &HashMap::from([ + ("bucket".into(), "-1".into()), + ("file.format".into(), "parquet".into()), + ("partition.default-name".into(), "it's null".into()), + ]), + "CTAS must preserve the supplied options without adding internal metadata" + ); + let definitions = ctx + .sql("SHOW CREATE TABLE result") + .await + .unwrap() + .collect() + .await + .unwrap(); + let definition = definitions[0] + .column(3) + .as_any() + .downcast_ref::() + .unwrap() + .value(0); + exec(&ctx, &definition.replace("\"result\"", "\"result_copy\"")).await; + exec( + &ctx, + "CREATE TABLE non_null AS WITH selected AS (SELECT part FROM source) SELECT part FROM selected WHERE part IS NOT NULL", + ) + .await; + let non_null = catalog + .get_table(&Identifier::new("default", "non_null")) + .await + .unwrap(); + assert!(non_null.schema().fields()[0].data_type().is_nullable()); + assert_eq!(row_count(&ctx, "SELECT * FROM non_null").await, 2); + let fresh = create_sql_context(catalog).await; + let rows = fresh + .sql("SELECT id, part FROM result ORDER BY id") + .await + .unwrap() + .collect() + .await + .unwrap(); + datafusion::assert_batches_eq!( + [ + "+----+------+", + "| id | part |", + "+----+------+", + "| 12 | |", + "| 13 | b |", + "+----+------+" + ], + &rows + ); +} + +#[tokio::test] +async fn empty_results_still_create_a_typed_table() { + let (_tmp, catalog) = create_test_env(); + let ctx = create_sql_context(catalog.clone()).await; + exec(&ctx, "CREATE TABLE empty COMMENT 'query result' AS SELECT CAST(NULL AS INT) AS id, CAST(1.25 AS DECIMAL(8, 2)) AS amount WHERE false").await; + let schema = catalog + .get_table(&Identifier::new("default", "empty")) + .await + .unwrap() + .schema() + .clone(); + assert_eq!(schema.fields().len(), 2); + assert_eq!(schema.comment(), Some("query result")); + assert_eq!(schema.fields()[0].name(), "id"); + assert_eq!( + schema.fields()[0].data_type(), + &DataType::Int(IntType::new()) + ); + assert_eq!(schema.fields()[1].name(), "amount"); + let DataType::Decimal(amount) = schema.fields()[1].data_type() else { + panic!("expected a decimal output"); + }; + assert_eq!((amount.precision(), amount.scale()), (8, 2)); + assert_eq!(row_count(&ctx, "SELECT * FROM empty").await, 0); +} + +#[tokio::test] +async fn time_precision_loss_is_rejected_before_table_creation() { + use arrow_array::types::Int32Type; + use arrow_array::{DictionaryArray, Time64MicrosecondArray, Time64NanosecondArray}; + + let (_tmp, catalog) = create_test_env(); + let ctx = create_sql_context(catalog.clone()).await; + let nanos = Arc::new(Time64NanosecondArray::from(vec![ + Some(45_296_123_456_789), + None, + ])); + let columns: Vec = vec![ + Arc::new(Time64MicrosecondArray::from(vec![ + Some(45_296_123_456), + None, + ])), + nanos.clone(), + Arc::new( + DictionaryArray::::try_new(Int32Array::from(vec![Some(0), None]), nanos) + .unwrap(), + ), + ]; + let schema = Arc::new(ArrowSchema::new( + ["micros", "nanos", "encoded"] + .into_iter() + .zip(&columns) + .map(|(name, column)| Field::new(name, column.data_type().clone(), true)) + .collect::>(), + )); + let batch = RecordBatch::try_new(schema.clone(), columns).unwrap(); + ctx.register_temp_table( + "time_source", + Arc::new(MemTable::try_new(schema, vec![vec![batch]]).unwrap()), + ) + .unwrap(); + for (index, (query, precision)) in [ + ("SELECT CAST('12:34:56.123456789' AS TIME) AS value", 9), + ("SELECT micros FROM time_source", 6), + ("SELECT nanos FROM time_source", 9), + ("SELECT encoded FROM time_source", 9), + ( + "SELECT named_struct('value', nanos) AS value FROM time_source", + 9, + ), + ("SELECT ARRAY[micros] AS value FROM time_source", 6), + ( + "SELECT MAP(ARRAY[1], ARRAY[nanos]) AS value FROM time_source", + 9, + ), + ( + "SELECT MAP(ARRAY[micros], ARRAY[1]) AS value FROM time_source", + 6, + ), + ( + "SELECT named_struct('values', ARRAY[encoded]) AS value FROM time_source", + 9, + ), + ("SELECT nanos FROM time_source WHERE false", 9), + ] + .iter() + .enumerate() + { + let target = format!("rejected_time_{index}"); + assert_sql_error( + &ctx, + &format!("CREATE TABLE {target} AS {query}"), + &format!("cannot preserve TIME({precision})"), + ) + .await; + assert!( + matches!( + catalog + .get_table(&Identifier::new("default", &target)) + .await, + Err(paimon::Error::TableNotExist { .. }) + ), + "{query}" + ); + } +} + +#[tokio::test] +async fn supported_time_outputs_preserve_values_and_nulls() { + use arrow_array::{Time32MillisecondArray, Time32SecondArray}; + + let (_tmp, catalog) = create_test_env(); + let ctx = create_sql_context(catalog.clone()).await; + let columns: Vec = vec![ + Arc::new(Time32SecondArray::from(vec![Some(45_296), None])), + Arc::new(Time32MillisecondArray::from(vec![Some(45_296_123), None])), + ]; + let schema = Arc::new(ArrowSchema::new( + ["seconds", "millis"] + .into_iter() + .zip(&columns) + .map(|(name, column)| Field::new(name, column.data_type().clone(), true)) + .collect::>(), + )); + let batch = RecordBatch::try_new(schema.clone(), columns).unwrap(); + ctx.register_temp_table( + "time_source", + Arc::new(MemTable::try_new(schema, vec![vec![batch]]).unwrap()), + ) + .unwrap(); + exec(&ctx, "CREATE TABLE supported_times AS SELECT seconds, millis, named_struct('value', millis) AS nested, ARRAY[seconds] AS items FROM time_source").await; + exec(&ctx, "CREATE TABLE time_text AS SELECT CAST(CAST('12:34:56.123456789' AS TIME) AS VARCHAR) AS value").await; + let fresh = create_sql_context(catalog).await; + let rows = fresh.sql("SELECT seconds, millis, nested.value, items[1] FROM supported_times ORDER BY seconds NULLS LAST") + .await.unwrap().collect().await.unwrap(); + for (index, expected) in [45_296_000, 45_296_123, 45_296_123, 45_296_000] + .into_iter() + .enumerate() + { + let values = rows + .iter() + .flat_map(|batch| { + batch + .column(index) + .as_any() + .downcast_ref::() + .unwrap() + .iter() + }) + .collect::>(); + assert_eq!(values, vec![Some(expected), None]); + } + let rows = fresh + .sql("SELECT value FROM time_text") + .await + .unwrap() + .collect() + .await + .unwrap(); + assert_eq!( + common::string_value(rows[0].column(0).as_ref(), 0), + "12:34:56.123456789" + ); +} + +#[tokio::test] +async fn quoted_names_and_cross_catalog_target_keep_the_session_source() { + let (_src, source) = create_test_env(); + let (_dst, destination) = create_test_env(); + let mut ctx = create_sql_context(source).await; + ctx.register_catalog("Other", destination.clone()) + .await + .unwrap(); + exec(&ctx, "CREATE DATABASE paimon.current_db").await; + exec(&ctx, "CREATE DATABASE \"Other\".\"TargetDb\"").await; + exec(&ctx, "USE paimon.current_db").await; + exec( + &ctx, + "CREATE TABLE \"Source.Table\" AS SELECT 7 AS \"Mixed.Id\"", + ) + .await; + exec( + &ctx, + "CREATE TABLE \"Other\".\"TargetDb\".\"Source.Table\" AS SELECT 99 AS \"Mixed.Id\"", + ) + .await; + exec( + &ctx, + r#"CREATE TABLE "Other"."TargetDb"."Result.Table" PARTITIONED BY ("Mixed.Id") + AS SELECT "Mixed.Id" FROM "Source.Table""#, + ) + .await; + assert_eq!( + query_ids( + &ctx, + r#"SELECT "Mixed.Id" FROM "Other"."TargetDb"."Result.Table""# + ) + .await, + [7] + ); + let result = destination + .get_table(&Identifier::new("TargetDb", "Result.Table")) + .await + .unwrap(); + assert_eq!(result.schema().partition_keys(), &["Mixed.Id"]); + exec( + &ctx, + r#"CREATE TABLE "PARTITIONED BY (id)" AS SELECT 2 AS version"#, + ) + .await; + exec( + &ctx, + r#"CREATE TABLE UPPER_TARGET AS SELECT version AS OF, $$PARTITIONED BY (id)$$ AS value + FROM "PARTITIONED BY (id)""#, + ) + .await; + assert_eq!( + row_count( + &ctx, + "SELECT * FROM upper_target WHERE of = 2 AND value = 'PARTITIONED BY (id)'" + ) + .await, + 1 + ); + exec( + &ctx, + "SET datafusion.sql_parser.enable_ident_normalization = false", + ) + .await; + exec( + &ctx, + "CREATE TABLE Other.TargetDb.CaseTarget PARTITIONED BY (Part) AS SELECT 8 AS Part", + ) + .await; + let table = destination + .get_table(&Identifier::new("TargetDb", "CaseTarget")) + .await + .unwrap(); + assert_eq!(table.schema().partition_keys(), &["Part"]); + assert_eq!( + row_count( + &ctx, + "SELECT Part FROM Other.TargetDb.CaseTarget WHERE Part = 8" + ) + .await, + 1 + ); + for (index, selector) in ["VERSION AS OF 1", "TIMESTAMP AS OF '2099-01-01 00:00:00'"] + .iter() + .enumerate() + { + exec( + &ctx, + &format!("CREATE TABLE Other.TargetDb.History{index} AS SELECT Part FROM Other.TargetDb.CaseTarget {selector}"), + ) + .await; + assert_eq!( + row_count( + &ctx, + &format!("SELECT Part FROM Other.TargetDb.History{index} WHERE Part = 8") + ) + .await, + 1 + ); + } +} + +#[tokio::test] +async fn time_travel_source_is_materialized_as_a_persistent_table() { + let (_tmp, catalog) = create_test_env(); + let ctx = create_sql_context(catalog.clone()).await; + exec(&ctx, "CREATE TABLE \"源表.db.rel.data\" AS SELECT 1 AS id").await; + catalog + .create_tag( + &Identifier::new("default", "源表.db.rel.data"), + "base", + Some(1), + false, + ) + .await + .unwrap(); + exec(&ctx, "INSERT INTO \"源表.db.rel.data\" VALUES (2)").await; + let fresh = create_sql_context(catalog.clone()).await; + for (table, selector, expected) in [ + ("historical", "VERSION AS OF 1", vec![1]), + ("tagged", "VERSION AS OF 'base'", vec![1]), + ( + "at_timestamp", + "TIMESTAMP AS OF '2099-01-01 00:00:00'", + vec![1, 2], + ), + ] { + exec( + &ctx, + &format!("; CREATE TABLE {table} PARTITIONED BY (id) AS SELECT \"源表.db.rel.data\".id FROM \"源表.db.rel.data\" {selector}"), + ) + .await; + let target = catalog + .get_table(&Identifier::new("default", table)) + .await + .unwrap(); + assert_eq!(target.schema().partition_keys(), &["id"]); + assert_eq!( + query_ids(&fresh, &format!("SELECT * FROM {table}")).await, + expected + ); + } + assert_eq!( + row_count(&fresh, "SELECT * FROM \"源表.db.rel.data\"").await, + 2 + ); + exec( + &ctx, + r#"CREATE TABLE joined AS + SELECT s1.id, $tag$source VERSION AS OF 99$tag$ AS note + FROM"源表.db.rel.data" VERSION AS /* tag */ OF 'base' AS s1 + JOIN "源表.db.rel.data" TIMESTAMP AS OF '2099-01-01 00:00:00' AS s2 ON s1.id = s2.id"#, + ) + .await; + assert_eq!( + query_ids( + &fresh, + "SELECT id FROM joined WHERE note = 'source VERSION AS OF 99'" + ) + .await, + [1] + ); + let query = "AS SELECT id FROM 源表.db.rel.data VERSION AS OF 1"; + assert_sql_error( + &ctx, + &format!("CREATE TABLE invalid_name {query}"), + "Expected 1, 2 or 3 parts, got 4", + ) + .await; + assert!(matches!( + catalog + .get_table(&Identifier::new("default", "invalid_name")) + .await, + Err(paimon::Error::TableNotExist { .. }) + )); + exec( + &ctx, + &format!("CREATE TABLE IF NOT EXISTS historical {query}"), + ) + .await; + exec(&ctx, "CREATE DATABASE db2").await; + exec(&ctx, "CREATE TABLE db2.src AS SELECT 10 AS id").await; + exec(&ctx, "CREATE TABLE src AS SELECT 12 AS id").await; + for (table, query) in [ + ("current_join", "SELECT paimon.default.src.id AS a, db2.src.id AS b FROM paimon.default.src CROSS JOIN db2.src"), + ("historical_join", "SELECT paimon.default.src.id AS a, db2.src.id AS b FROM paimon.default.src VERSION AS OF 1 CROSS JOIN db2.src VERSION AS OF 1"), + ("aliased_join", "SELECT a.id AS a, b.id AS b FROM paimon.default.src VERSION AS OF 1 AS a CROSS JOIN db2.src VERSION AS OF 1 AS b"), + ] { + exec(&ctx, &format!("CREATE TABLE {table} AS {query}")).await; + assert_eq!(row_count(&fresh, &format!("SELECT * FROM {table} WHERE a = 12 AND b = 10")).await, 1); + } + ctx.ctx().register_catalog( + "analytics", + Arc::new(datafusion::catalog::MemoryCatalogProvider::new()), + ); + exec(&ctx, "SET datafusion.catalog.default_catalog = 'analytics'").await; + exec( + &ctx, + "CREATE TABLE paimon.db2.history AS SELECT id FROM paimon.default.src VERSION AS OF 1", + ) + .await; + assert_eq!(query_ids(&fresh, "SELECT * FROM db2.history").await, [12]); +} + +#[tokio::test] +async fn inferred_nested_types_and_repeated_source_field_ids() { + let (_tmp, catalog) = create_test_env(); + let ctx = create_sql_context(catalog.clone()).await; + exec( + &ctx, + "CREATE TABLE source AS + SELECT named_struct('x', 1, 'text', 'hello') AS item, ARRAY[1, 2] AS items, + MAP(ARRAY['key'], ARRAY[3]) AS entries, CAST(1.25 AS DECIMAL(8, 2)) AS amount, + TIMESTAMP '2026-11-17 01:02:03' AS ts", + ) + .await; + exec( + &ctx, + "CREATE TABLE copied AS SELECT item AS first, item AS second, items, entries, amount, ts FROM source", + ) + .await; + let schema = catalog + .get_table(&Identifier::new("default", "copied")) + .await + .unwrap() + .schema() + .clone(); + assert_ne!(schema.fields()[0].id(), schema.fields()[1].id()); + if let (DataType::Row(first), DataType::Row(second)) = ( + schema.fields()[0].data_type(), + schema.fields()[1].data_type(), + ) { + assert_ne!(first.fields()[0].id(), second.fields()[0].id()); + } else { + panic!("expected nested ROW types"); + } + let rows = row_count( + &ctx, + "SELECT * FROM copied WHERE first.x = 1 AND second.text = 'hello' AND items[1] = 1 + AND entries['key'] = 3 AND amount = 1.25 AND ts = TIMESTAMP '2026-11-17 01:02:03'", + ) + .await; + assert_eq!(rows, 1); +} + +#[tokio::test] +async fn inferred_geospatial_types_preserve_crs_and_algorithm() { + let (_tmp, catalog) = create_test_env(); + let ctx = create_sql_context(catalog.clone()).await; + let wkb = "decode('0101000000000000000000F03F0000000000000040', 'hex')"; + for (name, predicate, expected_rows) in [("shapes", "true", 1), ("empty_shapes", "false", 0)] { + exec( + &ctx, + &format!( + "CREATE TABLE {name} AS SELECT + CAST({wkb} AS GEOMETRY('EPSG:3857')) AS geom, + CAST({wkb} AS GEOGRAPHY('OGC:CRS84', karney)) AS geog + WHERE {predicate}" + ), + ) + .await; + let table = catalog + .get_table(&Identifier::new("default", name)) + .await + .unwrap(); + assert!(matches!( + table.schema().fields()[0].data_type(), + DataType::Geometry(geometry) if geometry.crs() == "EPSG:3857" + )); + assert!(matches!( + table.schema().fields()[1].data_type(), + DataType::Geography(geography) + if geography.crs() == "OGC:CRS84" && geography.algorithm().to_string() == "karney" + )); + assert_eq!( + row_count( + &ctx, + &format!("SELECT * FROM {name} WHERE geom = {wkb} AND geog = {wkb}") + ) + .await, + expected_rows + ); + } +} + +#[tokio::test] +async fn arrow_layouts_are_normalized_for_scalar_and_nested_outputs() { + use arrow_array::builder::StringDictionaryBuilder; + use arrow_array::types::Int32Type; + use arrow_array::LargeStringArray; + + let (_tmp, catalog) = create_test_env(); + let ctx = create_sql_context(catalog).await; + let mut dictionary = StringDictionaryBuilder::::new(); + dictionary.append("encoded").unwrap(); + dictionary.append_null(); + let columns: Vec = vec![ + Arc::new(dictionary.finish()), + Arc::new(LargeStringArray::from(vec![Some("large"), None])), + ]; + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("encoded", columns[0].data_type().clone(), true), + Field::new("large", ArrowType::LargeUtf8, true), + ])); + let batch = RecordBatch::try_new(schema.clone(), columns).unwrap(); + ctx.register_temp_table( + "source", + Arc::new(MemTable::try_new(schema, vec![vec![batch]]).unwrap()), + ) + .unwrap(); + exec(&ctx, "CREATE TABLE decoded AS SELECT *, named_struct('encoded', encoded, 'large', large) AS nested, ARRAY[large] AS items FROM source").await; + assert_eq!( + row_count( + &ctx, + "SELECT * FROM decoded WHERE encoded = 'encoded' AND large = 'large' AND nested.encoded = 'encoded' AND nested.large = 'large' AND items[1] = 'large'" + ) + .await, + 1 + ); + assert_eq!( + row_count( + &ctx, + "SELECT * FROM decoded WHERE encoded IS NULL AND large IS NULL AND nested.encoded IS NULL AND nested.large IS NULL AND items[1] IS NULL" + ) + .await, + 1 + ); + use arrow_array::builder::{Int64Builder, MapBuilder, MapFieldNames, StringBuilder}; + use arrow_array::MapArray; + let names = MapFieldNames { + entry: "entries".into(), + key: "key".into(), + value: "value".into(), + }; + let mut map = MapBuilder::new(Some(names), StringBuilder::new(), Int64Builder::new()); + map.keys().append_value("discarded"); + map.values().append_value(0); + map.append(true).unwrap(); + map.keys().append_value("key"); + map.values().append_value(9); + map.append(true).unwrap(); + map.append(false).unwrap(); + let (field, offsets, entries, nulls, _) = map.finish().into_parts(); + for sorted in [false, true] { + let map = MapArray::new( + field.clone(), + offsets.clone(), + entries.clone(), + nulls.clone(), + sorted, + ) + .slice(1, 2); + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + "m", + map.data_type().clone(), + true, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(map)]).unwrap(); + ctx.register_temp_table( + format!("maps_{sorted}"), + Arc::new(MemTable::try_new(schema, vec![vec![batch]]).unwrap()), + ) + .unwrap(); + exec(&ctx, &format!("CREATE TABLE map_copy_{sorted} AS SELECT m, named_struct('m', m) AS nested, ARRAY[m] AS items FROM maps_{sorted}")).await; + assert_eq!(row_count(&ctx, &format!("SELECT * FROM map_copy_{sorted} WHERE m['key'] = 9 AND nested.m['key'] = 9 AND items[1]['key'] = 9")).await, 1); + assert_eq!(row_count(&ctx, &format!("SELECT * FROM map_copy_{sorted} WHERE m IS NULL AND nested.m IS NULL AND items[1] IS NULL")).await, 1); + } +} + +#[tokio::test] +async fn existing_target_is_a_noop_without_source_planning_or_execution() { + let (_tmp, catalog) = create_test_env(); + let ctx = create_sql_context(catalog.clone()).await; + exec(&ctx, "CREATE TABLE existing AS SELECT 9 AS id").await; + catalog + .create_table( + &Identifier::new("default", "external_target"), + Schema::builder() + .column("id", DataType::Int(IntType::new())) + .option("type", TableType::IcebergTable.as_str()) + .build() + .unwrap(), + false, + ) + .await + .unwrap(); + exec( + &ctx, + "CREATE TABLE object_target (id INT) WITH ('type' = 'object-table', 'path' = 'oss://bucket/objects')", + ) + .await; + for target in ["existing", "external_target", "object_target"] { + let identifier = Identifier::new("default", target); + let original = catalog.fetch_table_schema(&identifier).await.unwrap(); + for source in [ + "SELECT missing_function(id) FROM missing_source", + "SELECT * FROM missing_source VERSION AS OF 1", + "SELECT * FROM missing_source TIMESTAMP AS OF '2099-01-01 00:00:00'", + ] { + exec( + &ctx, + &format!("; CREATE TABLE IF NOT EXISTS {target} AS {source}"), + ) + .await; + } + assert_sql_error( + &ctx, + &format!("CREATE TABLE {target} AS SELECT * FROM missing_source"), + "already exists", + ) + .await; + assert_eq!( + catalog.fetch_table_schema(&identifier).await.unwrap(), + original + ); + } + assert_eq!(query_ids(&ctx, "SELECT * FROM existing").await, [9]); +} + +#[tokio::test] +async fn filesystem_target_lookup_errors_precede_source_planning() { + let (warehouse, catalog) = create_test_env(); + let ctx = create_sql_context(catalog).await; + exec(&ctx, "CREATE TABLE broken (id INT)").await; + std::fs::write( + warehouse.path().join("default.db/broken/schema/schema-0"), + b"invalid JSON", + ) + .unwrap(); + + for (target, expected_error) in [ + ("broken", "Failed to parse schema file"), + ("missing_db.target", "Database missing_db does not exist"), + ] { + for clause in ["", "IF NOT EXISTS "] { + assert_sql_error( + &ctx, + &format!("CREATE TABLE {clause}{target} AS SELECT * FROM missing_source"), + expected_error, + ) + .await; + } + } +} + +#[tokio::test] +async fn invalid_ctas_preserves_catalog_and_source_data() { + let (_tmp, catalog) = create_test_env(); + let ctx = create_sql_context(catalog.clone()).await; + exec(&ctx, "CREATE TABLE source AS SELECT 1 AS id").await; + ctx.register_temp_table( + "empty_columns", + Arc::new(MemTable::try_new(Arc::new(ArrowSchema::empty()), vec![vec![]]).unwrap()), + ) + .unwrap(); + for sql in [ + "CREATE TABLE bad AUTO_INCREMENT = 42 AS SELECT 1 AS id", + "CREATE TABLE bad DEFAULT CHARSET = utf8mb4 AS SELECT 1 AS id", + "CREATE TABLE bad COLLATE = utf8mb4_bin AS SELECT 1 AS id", + "CREATE TABLE bad WITH ('type' = 'iceberg-table') TBLPROPERTIES ('bucket' = '-1') AS SELECT 1 AS id", + "CREATE TABLE bad WITH ('bucket' = '0') OPTIONS ('file.format' = 'parquet') AS SELECT 1 AS id", + "CREATE TABLE bad WITH () OPTIONS ('bucket' = '-1') AS SELECT 1 AS id", + "CREATE TABLE bad AS SELECT * FROM empty_columns", + "CREATE TABLE bad AS SELECT NULL AS value", + "CREATE TABLE bad AS SELECT 1 AS id, 2 AS id", + "CREATE TABLE bad AS SELECT 1 AS \"_KEY_id\"", + "CREATE TABLE bad AS SELECT 1 AS alias VERSION AS OF 1", + "CREATE TABLE bad PARTITIONED BY (missing) AS SELECT 1 AS id", + r#"CREATE TABLE bad PARTITIONED BY (s.id) AS SELECT 1 AS "s.id""#, + "CREATE TABLE bad AS SELECT 1 AS id PARTITIONED BY (id)", + "CREATE TABLE bad WITH ('bucket' = '0') AS SELECT 1 AS id", + "CREATE EXTERNAL TABLE bad AS SELECT 1 AS id", + "CREATE TRANSIENT TABLE bad AS SELECT 1 AS id", + "CREATE GLOBAL TABLE bad AS SELECT 1 AS id", + "CREATE TABLE bad LOCATION '/tmp/not-a-ctas-table' AS SELECT 1 AS id", + "CREATE TABLE bad WITH ('type' = 'iceberg-table') AS SELECT 1 AS id", + "CREATE TABLE bad WITH ('type' = 'format-table') AS SELECT 1 AS id", + "CREATE TABLE bad (PRIMARY KEY (id)) AS SELECT 1 AS id", + ] { + assert!(ctx.sql(sql).await.is_err(), "{sql}"); + } + for (statement, expected_error) in [ + ( + r#"CREATE TABLE IF NOT EXISTS "source$branch_dev" AS SELECT * FROM missing_source"#, + "on Paimon branch 'dev' is not supported", + ), + ( + r#"CREATE TABLE "source$snapshots" AS SELECT * FROM missing_source"#, + "on Paimon system table 'snapshots' is not supported", + ), + ( + "CREATE OR REPLACE TABLE source AS SELECT 2 AS id", + "does not support modifiers before TABLE", + ), + ( + "CREATE OR REPLACE TABLE source PARTITIONED BY (id) AS SELECT * FROM missing_source VERSION AS OF 1", + "does not support modifiers before TABLE", + ), + ( + "CREATE OR ALTER TABLE source AS SELECT * FROM source TIMESTAMP AS OF '2099-01-01 00:00:00'", + "does not support modifiers before TABLE", + ), + ( + "CREATE TABLE source (id BIGINT) AS SELECT * FROM source VERSION AS OF 1", + "explicit column declarations", + ), + ( + "CREATE TABLE bad AS SELECT * FROM source VERSION AS OF 1; DROP TABLE source", + "Expected exactly one SQL statement", + ), + ] { + assert_sql_error(&ctx, &format!("/* start */ ; ; {statement}"), expected_error).await; + } + for sql in [ + "; CREATE TABLE bad AS SELECT id INTO side_effect FROM source VERSION AS OF 1", + "CREATE TABLE bad AS WITH changed AS (SELECT id INTO side_effect FROM source) SELECT * FROM changed", + "CREATE TABLE bad AS SELECT id FROM source UNION ALL SELECT id INTO side_effect FROM source", + "CREATE TABLE bad AS INSERT INTO source VALUES (2)", + "CREATE TABLE bad AS UPDATE source SET id = 2", + "CREATE TABLE bad AS DELETE FROM source", + "CREATE TABLE bad AS MERGE INTO source t USING (SELECT 1 AS id) s ON t.id = s.id WHEN MATCHED THEN DELETE", + "CREATE TABLE bad AS WITH changed AS (INSERT INTO source VALUES (2) RETURNING id) SELECT * FROM changed", + ] { + assert_sql_error(&ctx, sql, "requires a read-only query").await; + } + for missing in ["bad", "side_effect"] { + assert!(matches!( + catalog + .get_table(&Identifier::new("default", missing)) + .await, + Err(paimon::Error::TableNotExist { .. }) + )); + } + assert_eq!(query_ids(&ctx, "SELECT id FROM source").await, [1]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn rest_ctas_resolves_the_source_only_after_checking_the_target() { + let tmp = tempfile::tempdir().unwrap(); + let server = mock_server::start_mock_server( + "ctas".into(), + tmp.path().to_string_lossy().into_owned(), + paimon::api::ConfigResponse::new(HashMap::from([( + CatalogOptions::PREFIX.to_string(), + "mock-test".into(), + )])), + vec!["source_db".into(), "default".into()], + ) + .await; + let mut options = Options::new(); + options.set(CatalogOptions::URI, server.url().unwrap()); + options.set(CatalogOptions::WAREHOUSE, "ctas"); + options.set(CatalogOptions::TOKEN_PROVIDER, "bear"); + options.set(CatalogOptions::TOKEN, "test-token"); + let source = Arc::new(RESTCatalog::new(options, true).await.unwrap()); + let (_dest, destination) = create_test_env(); + let mut ctx = SQLContext::new(); + ctx.register_catalog("source", source).await.unwrap(); + ctx.register_catalog("dest", destination).await.unwrap(); + exec(&ctx, "USE source.source_db").await; + exec( + &ctx, + "CREATE FUNCTION plus_one(x BIGINT) RETURNS BIGINT RETURN x + 1", + ) + .await; + exec(&ctx, "CREATE TABLE inputs AS SELECT 8 AS id").await; + let query = "SELECT plus_one(id) AS id FROM inputs"; + server.add_table_with_schema( + "default", + "objects", + Schema::builder() + .column("id", DataType::Int(IntType::new())) + .option("type", TableType::ObjectTable.as_str()) + .build() + .unwrap(), + "oss://bucket/objects", + ); + exec( + &ctx, + "CREATE TABLE IF NOT EXISTS source.default.objects AS SELECT * FROM missing_source", + ) + .await; + assert_sql_error( + &ctx, + "CREATE TABLE source.default.objects AS SELECT * FROM missing_source", + "already exists", + ) + .await; + assert_eq!( + server.table_get_call_count("source_db", "missing_source"), + 0 + ); + server.add_no_permission_table("default", "forbidden"); + for (target, expected_error) in [ + ( + "source.missing_db.result", + "Database missing_db does not exist", + ), + ("source.default.forbidden", "No Permission"), + ] { + let before = server.table_get_call_count("source_db", "inputs"); + for clause in ["", "IF NOT EXISTS "] { + assert_sql_error( + &ctx, + &format!("CREATE TABLE {clause}{target} AS {query}"), + expected_error, + ) + .await; + } + assert_eq!(server.table_get_call_count("source_db", "inputs"), before); + } + for target in ["dest.default.result", "source.default.result"] { + let before = server.table_get_call_count("source_db", "inputs"); + exec( + &ctx, + &format!("CREATE TABLE {target} WITH ('commit.max-retries' = '0') AS {query}"), + ) + .await; + assert_eq!( + server.table_get_call_count("source_db", "inputs") - before, + 1 + ); + assert_eq!( + query_ids(&ctx, &format!("SELECT * FROM {target}")).await, + [9] + ); + } +} diff --git a/crates/integrations/datafusion/tests/create_table_as/execution.rs b/crates/integrations/datafusion/tests/create_table_as/execution.rs new file mode 100644 index 000000000..a89306eb7 --- /dev/null +++ b/crates/integrations/datafusion/tests/create_table_as/execution.rs @@ -0,0 +1,726 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use async_trait::async_trait; +use bytes::Bytes; +use datafusion::catalog::streaming::StreamingTable; +use datafusion::catalog::Session; +use datafusion::datasource::{TableProvider, TableType as DFTableType}; +use datafusion::error::{DataFusionError, Result as DFResult}; +use datafusion::execution::{SendableRecordBatchStream, TaskContext}; +use datafusion::logical_expr::{create_udf, ColumnarValue, Expr, Volatility}; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::streaming::{PartitionStream, StreamingTableExec}; +use datafusion::physical_plan::ExecutionPlan; +use futures::StreamExt; +use paimon::catalog::Database; +use paimon::io::{FileIOBuilder, FileIOProvider}; +use paimon::spec::SchemaChange; +use paimon::table::Table; +use paimon::FileSystemCatalog; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::time::Duration; +use tokio::sync::Notify; + +#[derive(Clone, Copy, Debug, PartialEq)] +enum Fault { + None, + Lookup, + PauseLookup, + Create, + LostCreateResponse, + Race, + LoadAfterCreate, + Write, + WriteAfterOutput, + BeforePublication, + LostResponse, + PausePublication, +} + +#[derive(Debug)] +struct InjectedIO { + operator: opendal::Operator, + fault: Fault, + published: AtomicBool, + data_paths: std::sync::Mutex>, + reached: Notify, + resume: Notify, +} + +fn injected_error(message: &str) -> paimon::Error { + paimon::Error::UnexpectedError { + message: message.into(), + source: None, + } +} + +fn relative(path: &str) -> &str { + path.trim_start_matches("file://").trim_start_matches('/') +} + +#[async_trait] +impl FileIOProvider for InjectedIO { + async fn create(&self, path: &str) -> paimon::Result<(opendal::Operator, String)> { + if path.ends_with(".parquet") { + let mut paths = self.data_paths.lock().unwrap(); + let index = paths + .iter() + .position(|entry| entry == path) + .unwrap_or_else(|| { + paths.push(path.into()); + paths.len() - 1 + }); + if self.fault == Fault::Write || self.fault == Fault::WriteAfterOutput && index == 1 { + return Err(injected_error("injected data write failure")); + } + } + if self.published.load(Ordering::SeqCst) + && self.fault == Fault::LostResponse + && path.contains("/snapshot/") + { + return Err(injected_error("injected snapshot read failure")); + } + Ok((self.operator.clone(), relative(path).into())) + } + + async fn write_atomic_if_not_exists( + &self, + path: &str, + bytes: Bytes, + ) -> paimon::Result> { + if path.contains("/snapshot/snapshot-") { + if self.fault == Fault::PausePublication { + self.reached.notify_one(); + self.resume.notified().await; + self.operator.write(relative(path), bytes).await.unwrap(); + self.published.store(true, Ordering::SeqCst); + return Ok(Some(true)); + } + if self.fault == Fault::BeforePublication { + return Err(injected_error("injected publication failure")); + } + if self.fault == Fault::LostResponse { + self.operator.write(relative(path), bytes).await.unwrap(); + self.published.store(true, Ordering::SeqCst); + return Err(injected_error("injected lost commit response")); + } + } + Ok(None) + } +} + +#[derive(Debug)] +struct TestCatalog { + inner: Arc, + fault: Fault, + creates: AtomicUsize, + drops: AtomicUsize, + io: Option>, + lookup_started: Notify, + resume_lookup: Notify, +} + +impl TestCatalog { + fn new(inner: Arc, fault: Fault) -> Arc { + let io = matches!( + fault, + Fault::Write + | Fault::WriteAfterOutput + | Fault::BeforePublication + | Fault::LostResponse + | Fault::PausePublication + ) + .then(|| { + Arc::new(InjectedIO { + operator: + opendal::Operator::from_config(opendal::services::MemoryConfig::default()) + .unwrap(), + fault, + published: AtomicBool::new(false), + data_paths: std::sync::Mutex::new(Vec::new()), + reached: Notify::new(), + resume: Notify::new(), + }) + }); + Arc::new(Self { + inner, + fault, + creates: AtomicUsize::new(0), + drops: AtomicUsize::new(0), + io, + lookup_started: Notify::new(), + resume_lookup: Notify::new(), + }) + } +} + +async fn append(table: &Table, value: i64) { + let builder = table.new_write_builder(); + let mut writer = builder.new_write().unwrap(); + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + "id", + ArrowType::Int64, + false, + )])); + writer + .write_arrow_batch( + &RecordBatch::try_new(schema, vec![Arc::new(Int64Array::from(vec![value]))]).unwrap(), + ) + .await + .unwrap(); + let messages = writer.prepare_commit().await.unwrap(); + builder.new_commit().commit(messages).await.unwrap(); +} + +#[async_trait] +impl Catalog for TestCatalog { + async fn list_databases(&self) -> paimon::Result> { + self.inner.list_databases().await + } + async fn create_database( + &self, + name: &str, + ignore: bool, + properties: HashMap, + ) -> paimon::Result<()> { + self.inner.create_database(name, ignore, properties).await + } + async fn get_database(&self, name: &str) -> paimon::Result { + self.inner.get_database(name).await + } + async fn drop_database(&self, name: &str, ignore: bool, cascade: bool) -> paimon::Result<()> { + self.inner.drop_database(name, ignore, cascade).await + } + async fn check_table_exists(&self, id: &Identifier) -> paimon::Result<()> { + if self.fault == Fault::Lookup { + return Err(injected_error("injected forbidden lookup")); + } + if self.fault == Fault::PauseLookup + && id.object() == "target" + && self.creates.load(Ordering::SeqCst) == 0 + { + self.lookup_started.notify_one(); + self.resume_lookup.notified().await; + } + self.inner.check_table_exists(id).await + } + async fn get_table(&self, id: &Identifier) -> paimon::Result { + if self.fault == Fault::LoadAfterCreate && self.creates.load(Ordering::SeqCst) > 0 { + return Err(injected_error("injected table load failure")); + } + let table = self.inner.get_table(id).await?; + if let Some(io) = &self.io { + let schema_path = format!("{}/schema/schema-0", table.location()); + if !io.operator.exists(relative(&schema_path)).await.unwrap() { + io.operator + .write( + relative(&schema_path), + serde_json::to_vec(table.schema()).unwrap(), + ) + .await + .unwrap(); + } + Ok(Table::new( + FileIOBuilder::new("memory") + .with_provider(io.clone()) + .build()?, + id.clone(), + table.location().to_string(), + table.schema().clone(), + None, + )) + } else { + Ok(table) + } + } + async fn list_tables(&self, db: &str) -> paimon::Result> { + self.inner.list_tables(db).await + } + async fn create_table( + &self, + id: &Identifier, + schema: Schema, + ignore: bool, + ) -> paimon::Result<()> { + self.inner.create_table(id, schema, ignore).await + } + async fn create_table_exclusive(&self, id: &Identifier, schema: Schema) -> paimon::Result<()> { + self.creates.fetch_add(1, Ordering::SeqCst); + if self.fault == Fault::Create { + return Err(injected_error("injected table creation failure")); + } + if self.fault == Fault::Race { + self.inner.create_table(id, schema.clone(), false).await?; + append(&self.inner.get_table(id).await?, 77).await; + } + self.inner.create_table_exclusive(id, schema).await?; + if self.fault == Fault::LostCreateResponse { + return Err(injected_error("injected lost create response")); + } + Ok(()) + } + async fn drop_table(&self, id: &Identifier, ignore: bool) -> paimon::Result<()> { + self.drops.fetch_add(1, Ordering::SeqCst); + self.inner.drop_table(id, ignore).await + } + async fn rename_table( + &self, + from: &Identifier, + to: &Identifier, + ignore: bool, + ) -> paimon::Result<()> { + self.inner.rename_table(from, to, ignore).await + } + async fn alter_table( + &self, + id: &Identifier, + changes: Vec, + ignore: bool, + ) -> paimon::Result<()> { + self.inner.alter_table(id, changes, ignore).await + } +} + +#[derive(Debug)] +struct ProbeSource { + schema: Arc, + scans: AtomicUsize, + executions: Arc, + fail: bool, + concurrent_writer: Option>, +} +impl ProbeSource { + fn new(fail: bool, concurrent_writer: Option>) -> Arc { + Arc::new(Self { + schema: Arc::new(ArrowSchema::new(vec![Field::new( + "id", + ArrowType::Int64, + false, + )])), + scans: AtomicUsize::new(0), + executions: Arc::new(AtomicUsize::new(0)), + fail, + concurrent_writer, + }) + } +} +#[derive(Debug)] +struct ProbePartition { + schema: Arc, + executions: Arc, + fail: bool, + concurrent_writer: Option>, +} +impl PartitionStream for ProbePartition { + fn schema(&self) -> &Arc { + &self.schema + } + fn execute(&self, _: Arc) -> SendableRecordBatchStream { + self.executions.fetch_add(1, Ordering::SeqCst); + let first = RecordBatch::try_new( + self.schema.clone(), + vec![Arc::new(Int64Array::from(vec![11]))], + ) + .unwrap(); + let concurrent_writer = self.concurrent_writer.clone(); + let fail = self.fail; + let schema = self.schema.clone(); + let stream = + futures::stream::once(async { Ok(first) }).chain(futures::stream::once(async move { + if let Some(catalog) = concurrent_writer { + let table = catalog + .get_table(&Identifier::new("default", "target")) + .await + .unwrap(); + append(&table, 88).await; + } + if fail { + Err(DataFusionError::Execution("injected source failure".into())) + } else { + Ok(RecordBatch::try_new( + schema, + vec![Arc::new(Int64Array::from(vec![12]))], + )?) + } + })); + Box::pin(RecordBatchStreamAdapter::new(self.schema.clone(), stream)) + } +} +#[async_trait] +impl TableProvider for ProbeSource { + fn schema(&self) -> Arc { + self.schema.clone() + } + fn table_type(&self) -> DFTableType { + DFTableType::Base + } + async fn scan( + &self, + _: &dyn Session, + projection: Option<&Vec>, + _: &[Expr], + limit: Option, + ) -> DFResult> { + self.scans.fetch_add(1, Ordering::SeqCst); + Ok(Arc::new(StreamingTableExec::try_new( + self.schema.clone(), + vec![Arc::new(ProbePartition { + schema: self.schema.clone(), + executions: self.executions.clone(), + fail: self.fail, + concurrent_writer: self.concurrent_writer.clone(), + })], + projection, + [], + false, + limit, + )?)) + } +} + +async fn context(catalog: Arc, source: Arc) -> SQLContext { + let mut ctx = SQLContext::new(); + ctx.register_catalog("paimon", catalog).await.unwrap(); + ctx.register_temp_table("source", source).unwrap(); + ctx +} + +#[tokio::test] +async fn inference_and_population_use_one_plan_in_the_statement_namespace() { + let (_tmp, inner) = create_test_env(); + let catalog = TestCatalog::new(inner, Fault::PauseLookup); + let source = ProbeSource::new(false, None); + let ctx = context(catalog.clone(), source.clone()).await; + exec(&ctx, "CREATE DATABASE other").await; + ctx.register_temp_table("other.source", ProbeSource::new(true, None)) + .unwrap(); + let observed_rows = Arc::new(AtomicUsize::new(0)); + let counter = observed_rows.clone(); + ctx.ctx().register_udf(create_udf( + "observe", + vec![ArrowType::Int64], + ArrowType::Int64, + Volatility::Volatile, + Arc::new(move |args| { + let ColumnarValue::Array(ids) = &args[0] else { + panic!("expected the source column"); + }; + counter.fetch_add(ids.len(), Ordering::SeqCst); + Ok(args[0].clone()) + }), + )); + let query = "SELECT observe(id) AS id FROM source"; + let sql = format!("CREATE TABLE target AS {query}"); + tokio::time::timeout(Duration::from_secs(10), async { + tokio::join!(exec(&ctx, &sql), async { + catalog.lookup_started.notified().await; + ctx.set_current_database("other").await.unwrap(); + catalog.resume_lookup.notify_one(); + }); + }) + .await + .unwrap(); + assert_eq!(source.scans.load(Ordering::SeqCst), 1); + assert_eq!(source.executions.load(Ordering::SeqCst), 1); + assert_eq!( + query_ids(&ctx, "SELECT * FROM default.target").await, + [11, 12] + ); + assert_eq!(catalog.creates.load(Ordering::SeqCst), 1); + exec( + &ctx, + &format!("CREATE TABLE IF NOT EXISTS default.target AS {query}"), + ) + .await; + assert_eq!(source.scans.load(Ordering::SeqCst), 1); + assert_eq!(source.executions.load(Ordering::SeqCst), 1); + assert_eq!(observed_rows.load(Ordering::SeqCst), 2); + assert_eq!(catalog.creates.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn create_race_never_populates_or_drops_the_winner() { + for if_not_exists in [false, true] { + let (_tmp, inner) = create_test_env(); + let catalog = TestCatalog::new(inner.clone(), Fault::Race); + let source = ProbeSource::new(false, None); + let ctx = context(catalog.clone(), source.clone()).await; + let clause = if if_not_exists { "IF NOT EXISTS " } else { "" }; + let result = ctx + .sql(&format!( + "CREATE TABLE {clause}target AS SELECT * FROM source" + )) + .await; + assert_eq!(result.is_ok(), if_not_exists); + if let Err(error) = result { + assert!(error.to_string().contains("already exists"), "{error}"); + } + assert_eq!(catalog.creates.load(Ordering::SeqCst), 1); + assert_eq!(source.executions.load(Ordering::SeqCst), 0); + assert_eq!(catalog.drops.load(Ordering::SeqCst), 0); + assert_eq!(query_ids(&ctx, "SELECT * FROM target").await, [77]); + } +} + +#[tokio::test] +async fn catalog_failures_do_not_execute_the_source_or_drop_the_target() { + for (fault, expected_error) in [ + (Fault::Lookup, "injected forbidden lookup"), + (Fault::Create, "injected table creation failure"), + (Fault::LostCreateResponse, "injected lost create response"), + (Fault::LoadAfterCreate, "injected table load failure"), + ] { + for if_not_exists in [false, true] { + let (_tmp, inner) = create_test_env(); + let catalog = TestCatalog::new(inner.clone(), fault); + let source = ProbeSource::new(false, None); + let ctx = context(catalog.clone(), source.clone()).await; + let clause = if if_not_exists { "IF NOT EXISTS " } else { "" }; + let sql = format!("CREATE TABLE {clause}target AS SELECT * FROM source"); + let error = ctx.sql(&sql).await.unwrap_err().to_string(); + assert!(error.contains(expected_error), "{error}"); + if fault == Fault::LoadAfterCreate { + assert!( + error.contains("Failed to load newly created table 'paimon.default.target'"), + "{error}" + ); + } + let planned = usize::from(fault != Fault::Lookup); + assert_eq!(catalog.creates.load(Ordering::SeqCst), planned); + assert_eq!(source.scans.load(Ordering::SeqCst), planned); + assert_eq!(source.executions.load(Ordering::SeqCst), 0); + assert_eq!(catalog.drops.load(Ordering::SeqCst), 0); + let target = inner.get_table(&Identifier::new("default", "target")).await; + if matches!(fault, Fault::LostCreateResponse | Fault::LoadAfterCreate) { + assert!(target + .unwrap() + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .is_none()); + } else { + assert!(matches!(target, Err(paimon::Error::TableNotExist { .. }))); + } + } + } +} + +#[tokio::test] +async fn population_preserves_concurrent_rows_and_cleans_failed_output() { + for (fail, concurrent) in [(true, false), (true, true), (false, true)] { + let (tmp, inner) = create_test_env(); + let catalog = TestCatalog::new(inner.clone(), Fault::None); + let source = ProbeSource::new(fail, concurrent.then_some(inner)); + let ctx = context(catalog.clone(), source.clone()).await; + let sql = "CREATE TABLE target WITH ('target-file-size' = '1 b') AS SELECT * FROM source"; + if fail { + assert_sql_error(&ctx, sql, "injected source failure").await; + } else { + exec(&ctx, sql).await; + } + assert_eq!(catalog.drops.load(Ordering::SeqCst), 0); + let mut expected = if fail { vec![] } else { vec![11, 12] }; + if concurrent { + expected.push(88); + } + assert_eq!(query_ids(&ctx, "SELECT * FROM target").await, expected); + let data_files = common::files_under(tmp.path()) + .iter() + .filter(|path| path.extension().is_some_and(|ext| ext == "parquet")) + .count(); + assert_eq!( + data_files, + expected.len(), + "only committed output should remain: fail={fail}, concurrent={concurrent}" + ); + exec( + &ctx, + "CREATE TABLE IF NOT EXISTS target AS SELECT * FROM source", + ) + .await; + assert_eq!(catalog.creates.load(Ordering::SeqCst), 1); + assert_eq!(source.executions.load(Ordering::SeqCst), 1); + } +} + +#[tokio::test] +async fn write_and_publication_failures_keep_the_table_and_uncertain_files() { + for fault in [ + Fault::Write, + Fault::WriteAfterOutput, + Fault::BeforePublication, + Fault::LostResponse, + ] { + let (_tmp, inner) = create_test_env(); + let catalog = TestCatalog::new(inner, fault); + let source = ProbeSource::new(false, None); + let ctx = context(catalog.clone(), source.clone()).await; + let error = ctx + .sql("CREATE TABLE target WITH ('target-file-size' = '1 b', 'commit.max-retries' = '0') AS SELECT * FROM source") + .await + .expect_err("injected failure") + .to_string(); + assert!(error.contains("injected"), "{fault:?}: {error}"); + assert_eq!(catalog.drops.load(Ordering::SeqCst), 0); + catalog + .inner + .get_table(&Identifier::new("default", "target")) + .await + .unwrap(); + let io = catalog.io.as_ref().unwrap(); + let entries = io.operator.list_with("/").recursive(true).await.unwrap(); + let data = entries + .iter() + .filter(|entry| entry.path().ends_with(".parquet")) + .count(); + if matches!(fault, Fault::Write | Fault::WriteAfterOutput) { + assert_eq!(data, 0); + assert_eq!( + io.data_paths.lock().unwrap().len(), + if fault == Fault::Write { 1 } else { 2 } + ); + assert!(error.contains("no commit was attempted"), "{error}"); + } else { + assert!( + data > 0, + "prepared files must survive uncertain publication" + ); + assert!(error.contains("publication may have succeeded"), "{error}"); + } + assert_eq!( + io.published.load(Ordering::SeqCst), + fault == Fault::LostResponse + ); + exec( + &ctx, + "CREATE TABLE IF NOT EXISTS target AS SELECT * FROM source", + ) + .await; + assert_eq!(catalog.creates.load(Ordering::SeqCst), 1); + assert_eq!(source.executions.load(Ordering::SeqCst), 1); + if fault == Fault::LostResponse { + io.published.store(false, Ordering::SeqCst); + assert_eq!(query_ids(&ctx, "SELECT * FROM target").await, [11, 12]); + } + } +} + +#[derive(Debug)] +struct PausedSource(RecordBatch); + +impl PartitionStream for PausedSource { + fn schema(&self) -> &Arc { + self.0.schema_ref() + } + fn execute(&self, _: Arc) -> SendableRecordBatchStream { + let input = futures::stream::iter([Ok(self.0.clone())]).chain(futures::stream::pending()); + Box::pin(RecordBatchStreamAdapter::new(self.0.schema(), input)) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cancellation_discards_uncommitted_output() { + let (tmp, catalog) = create_test_env(); + let ctx = create_sql_context(catalog.clone()).await; + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + "id", + ArrowType::Int64, + false, + )])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int64Array::from(vec![12; 24_576]))], + ) + .unwrap(); + let table = StreamingTable::try_new(schema, vec![Arc::new(PausedSource(batch))]).unwrap(); + ctx.register_temp_table("probe", Arc::new(table)).unwrap(); + let statement = tokio::spawn(async move { + ctx.sql("CREATE TABLE result WITH ('target-file-size' = '1 b') AS SELECT id FROM probe") + .await + .map(|_| ()) + }); + let outputs = || { + common::files_under(tmp.path()) + .into_iter() + .filter(|p| p.extension().is_some_and(|e| e == "parquet")) + .collect::>() + }; + tokio::time::timeout(Duration::from_secs(10), async { + while !outputs() + .iter() + .any(|path| path.metadata().is_ok_and(|metadata| metadata.len() > 0)) + { + tokio::time::sleep(Duration::from_millis(15)).await; + } + }) + .await + .unwrap(); + statement.abort(); + assert!(statement.await.unwrap_err().is_cancelled()); + tokio::time::timeout(Duration::from_secs(10), async { + while !outputs().is_empty() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert!(outputs().is_empty(), "orphan files: {:?}", outputs()); + let table = catalog + .get_table(&Identifier::new("default", "result")) + .await + .unwrap(); + assert!(table + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .is_none()); + let fresh = create_sql_context(catalog).await; + assert_eq!(row_count(&fresh, "SELECT * FROM result").await, 0); +} + +#[tokio::test] +async fn cancellation_finishes_started_publication() { + let (_tmp, inner) = create_test_env(); + let catalog = TestCatalog::new(inner, Fault::PausePublication); + let ctx = Arc::new(context(catalog.clone(), ProbeSource::new(false, None)).await); + let writer_ctx = ctx.clone(); + let statement = tokio::spawn(async move { + writer_ctx + .sql("CREATE TABLE target AS SELECT * FROM source") + .await + .map(|_| ()) + }); + let io = catalog.io.as_ref().unwrap(); + tokio::time::timeout(Duration::from_secs(10), io.reached.notified()) + .await + .unwrap(); + statement.abort(); + assert!(statement.await.unwrap_err().is_cancelled()); + io.resume.notify_one(); + tokio::time::timeout(Duration::from_secs(10), async { + while !io.published.load(Ordering::SeqCst) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert_eq!(query_ids(&ctx, "SELECT id FROM target").await, [11, 12]); +} diff --git a/crates/integrations/datafusion/tests/read_files_behavior.rs b/crates/integrations/datafusion/tests/read_files_behavior.rs index 6cba4097c..c4f9c16da 100644 --- a/crates/integrations/datafusion/tests/read_files_behavior.rs +++ b/crates/integrations/datafusion/tests/read_files_behavior.rs @@ -291,10 +291,15 @@ async fn wildcard_replacements_preserve_aliases_for_outer_queries_and_ctas() { ) .await .unwrap(); - ctx.sql("CREATE TEMPORARY TABLE replaced AS SELECT * REPLACE (upper(value) AS value) FROM read_files('memory://files/data.txt')").await.unwrap(); - assert_eq!(columns(&ctx, "SELECT * FROM replaced").await, ["value"]); - let values = rows(&ctx, "SELECT value FROM replaced").await; - assert_eq!(values[0][0].try_as_str().flatten(), Some("HELLO")); + for (modifier, table) in [("TEMPORARY ", "temp_replaced"), ("", "replaced")] { + ctx.sql(&format!("CREATE {modifier}TABLE {table} AS SELECT * REPLACE (upper(value) AS value) FROM read_files('memory://files/data.txt')")).await.unwrap(); + assert_eq!( + columns(&ctx, &format!("SELECT * FROM {table}")).await, + ["value"] + ); + let values = rows(&ctx, &format!("SELECT value FROM {table}")).await; + assert_eq!(values[0][0].try_as_str().flatten(), Some("HELLO")); + } } #[tokio::test] diff --git a/crates/paimon/src/catalog/filesystem.rs b/crates/paimon/src/catalog/filesystem.rs index 2f2b7eee3..d35f99b97 100644 --- a/crates/paimon/src/catalog/filesystem.rs +++ b/crates/paimon/src/catalog/filesystem.rs @@ -26,7 +26,7 @@ use crate::catalog::{Catalog, Database, Identifier, DB_LOCATION_PROP, DB_SUFFIX} use crate::common::{CatalogOptions, Options}; use crate::error::{ConfigInvalidSnafu, Error, Result}; use crate::io::cache::{create_local_cache, LocalCache}; -use crate::io::FileIO; +use crate::io::{AtomicPublication, FileIO}; use crate::spec::{ CoreOptions, Schema, Snapshot, TableSchema, TableType, INDEX_FILE_IN_DATA_FILE_DIR_OPTION, PRIMARY_KEY_NULLABLE_OPTION, TABLE_TYPE_OPTION, @@ -47,6 +47,15 @@ fn make_path(parent: &str, child: &str) -> String { format!("{parent}/{child}") } +fn serialize_table_schema(schema: &TableSchema) -> Result { + serde_json::to_vec(schema) + .map(Bytes::from) + .map_err(|error| Error::DataInvalid { + message: format!("Failed to serialize schema: {error}"), + source: Some(Box::new(error)), + }) +} + /// Filesystem catalog implementation. /// /// This catalog stores metadata on a filesystem with the following structure: @@ -259,14 +268,60 @@ impl FileSystemCatalog { self.file_io.mkdirs(&schema_dir).await?; let schema_path = self.schema_file_path(table_path, schema.id()); let output_file = self.file_io.new_output(&schema_path)?; - let content = - Bytes::from( - serde_json::to_string(schema).map_err(|e| Error::DataInvalid { - message: format!("Failed to serialize schema: {e}"), - source: Some(Box::new(e)), - })?, - ); - output_file.write(content).await?; + output_file.write(serialize_table_schema(schema)?).await + } + + async fn create_table_with_publication( + &self, + identifier: &Identifier, + mut creation: Schema, + ignore_if_exists: bool, + publication: AtomicPublication, + ) -> Result<()> { + identifier.validate()?; + let declared = CoreOptions::new(creation.options()).table_type()?; + + let table_path = self.table_path(identifier); + if declared == TableType::ObjectTable { + creation = ObjectTable::normalize_creation(&creation, &table_path)?; + } + + let table_exists = self.table_exists(identifier).await?; + + if !ignore_if_exists && table_exists { + return Err(Error::TableAlreadyExist { + full_name: identifier.full_name(), + }); + } + + if !self.database_exists(identifier.database()).await? { + return Err(Error::DatabaseNotExist { + database: identifier.database().to_string(), + }); + } + + if !table_exists { + // Another catalog may have dropped a table at this path with cached schema or snapshots. + self.file_io.invalidate_directory_cache(&table_path).await?; + self.file_io + .mkdirs(&self.schema_dir_path(&table_path)) + .await?; + let table_schema = TableSchema::new(0, &creation); + let created = self + .file_io + .write_new_file( + &self.schema_file_path(&table_path, 0), + serialize_table_schema(&table_schema)?, + publication, + ) + .await?; + if !created && !ignore_if_exists { + return Err(Error::TableAlreadyExist { + full_name: identifier.full_name(), + }); + } + } + Ok(()) } @@ -379,6 +434,17 @@ impl Catalog for FileSystemCatalog { Ok(()) } + async fn check_table_exists(&self, identifier: &Identifier) -> Result<()> { + match self.fetch_table_schema(identifier).await { + Ok(_) => Ok(()), + Err(error @ Error::TableNotExist { .. }) => { + self.get_database(identifier.database()).await?; + Err(error) + } + Err(error) => Err(error), + } + } + async fn get_table(&self, identifier: &Identifier) -> Result
{ let (table_path, schema) = self.fetch_table_schema(identifier).await?; self.build_table(identifier, table_path, schema) @@ -437,40 +503,25 @@ impl Catalog for FileSystemCatalog { async fn create_table( &self, identifier: &Identifier, - mut creation: Schema, + creation: Schema, ignore_if_exists: bool, ) -> Result<()> { - identifier.validate()?; - // Never persist a type nothing can load. - let declared = CoreOptions::new(creation.options()).table_type()?; - - let table_path = self.table_path(identifier); - if declared == TableType::ObjectTable { - creation = ObjectTable::normalize_creation(&creation, &table_path)?; - } - - let table_exists = self.table_exists(identifier).await?; - - if !ignore_if_exists && table_exists { - return Err(Error::TableAlreadyExist { - full_name: identifier.full_name(), - }); - } - - if !self.database_exists(identifier.database()).await? { - return Err(Error::DatabaseNotExist { - database: identifier.database().to_string(), - }); - } - - // todo: consider with lock - if !table_exists { - self.file_io.mkdirs(&table_path).await?; - let table_schema = TableSchema::new(0, &creation); - self.save_table_schema(&table_path, &table_schema).await?; - } + self.create_table_with_publication( + identifier, + creation, + ignore_if_exists, + AtomicPublication::IfSupported, + ) + .await + } - Ok(()) + async fn create_table_exclusive( + &self, + identifier: &Identifier, + creation: Schema, + ) -> Result<()> { + self.create_table_with_publication(identifier, creation, false, AtomicPublication::Required) + .await } async fn drop_table(&self, identifier: &Identifier, ignore_if_not_exists: bool) -> Result<()> { @@ -857,19 +908,50 @@ mod tests { FileSystemCatalog::new(options).unwrap() } - #[test] - fn test_filesystem_catalog_builds_local_cache_from_catalog_options() { - let warehouse = TempDir::new().unwrap(); + #[tokio::test] + async fn test_cached_catalog_reads_the_recreated_table_schema() { + let (warehouse, other) = create_test_catalog(); + let cache_dir = TempDir::new().unwrap(); let mut options = Options::new(); options.set( CatalogOptions::WAREHOUSE, warehouse.path().to_string_lossy(), ); options.set(CatalogOptions::LOCAL_CACHE_ENABLED, "true"); + options.set( + CatalogOptions::LOCAL_CACHE_DIR, + cache_dir.path().to_string_lossy(), + ); let catalog = FileSystemCatalog::new(options).unwrap(); - assert!(catalog.file_io().has_local_cache()); + let id = Identifier::new("db", "t"); + other + .create_database("db", false, HashMap::new()) + .await + .unwrap(); + other + .create_table(&id, testing_schema(), false) + .await + .unwrap(); + assert_eq!( + catalog.get_table(&id).await.unwrap().schema().fields()[0].name(), + "id" + ); + + other.drop_table(&id, false).await.unwrap(); + let schema = Schema::builder() + .column( + "replacement", + crate::spec::DataType::Int(crate::spec::IntType::new()), + ) + .build() + .unwrap(); + catalog.create_table_exclusive(&id, schema).await.unwrap(); + assert_eq!( + catalog.get_table(&id).await.unwrap().schema().fields()[0].name(), + "replacement" + ); } #[test] @@ -1752,6 +1834,205 @@ mod tests { ); } + #[cfg(feature = "storage-fs")] + #[tokio::test] + async fn test_table_creation_without_atomic_publication() { + use futures::TryStreamExt; + + let (temp, mut catalog) = create_test_catalog(); + let mut config = opendal_service_fs::FsConfig::default(); + config.root = Some("/".into()); + let operator = opendal::Operator::from_config(config).unwrap().layer( + opendal::layers::CapabilityOverrideLayer::new(|mut cap| { + cap.rename_with_if_not_exists = false; + cap.write_with_if_not_exists = false; + cap + }), + ); + catalog.file_io = crate::io::FileIOBuilder::new("file") + .with_fs_operator(operator) + .build() + .unwrap(); + catalog + .create_database("db", false, HashMap::new()) + .await + .unwrap(); + + let data = temp.path().join("existing"); + std::fs::create_dir(&data).unwrap(); + let batch = arrow_array::record_batch!(("id", Int32, [42])).unwrap(); + let file = std::fs::File::create(data.join("part.parquet")).unwrap(); + let mut writer = parquet::arrow::ArrowWriter::try_new(file, batch.schema(), None).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + let identifier = Identifier::new("db", "registered"); + assert!(matches!( + catalog + .create_table_exclusive(&identifier, testing_schema()) + .await, + Err(Error::IoUnsupported { .. }) + )); + assert!(matches!( + catalog.get_table(&identifier).await, + Err(Error::TableNotExist { .. }) + )); + let schema = Schema::builder() + .column("id", DataType::Int(IntType::new())) + .option("type", "format-table") + .option("file.format", "parquet") + .option("path", data.to_str().unwrap()) + .build() + .unwrap(); + catalog + .create_table(&identifier, schema, false) + .await + .unwrap(); + catalog + .create_table(&identifier, testing_schema(), true) + .await + .unwrap(); + assert!(matches!( + catalog.create_table(&identifier, testing_schema(), false).await, + Err(Error::TableAlreadyExist { full_name }) if full_name == identifier.full_name() + )); + let table = catalog.get_table(&identifier).await.unwrap(); + let builder = table.new_read_builder(); + let plan = builder.new_scan().plan().await.unwrap(); + let batches = builder + .new_read() + .unwrap() + .to_arrow(plan.splits()) + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].columns(), batch.columns()); + + assert!(matches!( + catalog.create_table_exclusive(&identifier, testing_schema()).await, + Err(Error::TableAlreadyExist { full_name }) if full_name == identifier.full_name() + )); + } + + #[tokio::test] + async fn test_create_table_propagates_a_failed_conditional_rename() { + let (_warehouse, mut catalog) = create_test_catalog(); + let operator = opendal::Operator::from_config(opendal::services::MemoryConfig::default()) + .unwrap() + .layer(opendal::layers::CapabilityOverrideLayer::new(|mut cap| { + cap.rename = true; + cap.rename_with_if_not_exists = true; + cap + })); + catalog.file_io = crate::io::FileIOBuilder::new("file") + .with_fs_operator(operator) + .build() + .unwrap(); + catalog + .create_database("db", false, HashMap::new()) + .await + .unwrap(); + let id = Identifier::new("db", "t"); + let schema_dir = catalog.schema_dir_path(&catalog.table_path(&id)); + let other_staged_schema = format!("{schema_dir}/schema-0.tmp-other"); + catalog + .file_io + .new_output(&other_staged_schema) + .unwrap() + .write(Bytes::from_static(b"other creator")) + .await + .unwrap(); + for create in [ + catalog.create_table(&id, testing_schema(), false), + catalog.create_table(&id, testing_schema(), true), + catalog.create_table_exclusive(&id, testing_schema()), + ] { + let error = create.await.unwrap_err(); + assert!( + matches!(error, Error::IoUnexpected { ref source, .. } if source.kind() == opendal::ErrorKind::Unsupported) + ); + assert!(matches!( + catalog.get_table(&id).await, + Err(Error::TableNotExist { .. }) + )); + let files = catalog.file_io.list_status(&schema_dir).await.unwrap(); + assert_eq!( + files.iter().map(|file| &file.path).collect::>(), + [&other_staged_schema] + ); + } + } + + #[tokio::test] + async fn test_create_table_atomically_rejects_a_raced_creator() { + #[derive(Debug)] + struct SharedMemoryStorage { + operator: opendal::Operator, + publication: tokio::sync::Barrier, + } + + #[async_trait] + impl crate::io::FileIOProvider for SharedMemoryStorage { + async fn create(&self, path: &str) -> Result<(opendal::Operator, String)> { + Ok(( + self.operator.clone(), + path.trim_start_matches("memory:/").into(), + )) + } + + async fn write_atomic_if_not_exists( + &self, + _path: &str, + _bytes: Bytes, + ) -> Result> { + self.publication.wait().await; + Ok(None) + } + } + + for ignore_if_exists in [false, true] { + let storage = std::sync::Arc::new(SharedMemoryStorage { + operator: + opendal::Operator::from_config(opendal::services::MemoryConfig::default()) + .unwrap(), + publication: tokio::sync::Barrier::new(2), + }); + let mut first = create_memory_catalog(); + let mut second = create_memory_catalog(); + first.file_io = first.file_io.clone().with_provider(storage.clone()); + second.file_io = second.file_io.clone().with_provider(storage); + first + .create_database("db", false, HashMap::new()) + .await + .unwrap(); + let identifier = Identifier::new("db", "raced"); + let schema = |name: &str| { + Schema::builder() + .column(name, DataType::Int(IntType::new())) + .build() + .unwrap() + }; + let (a, b) = tokio::time::timeout(std::time::Duration::from_secs(10), async { + tokio::join!( + first.create_table_exclusive(&identifier, schema("first")), + second.create_table(&identifier, schema("second"), ignore_if_exists), + ) + }) + .await + .unwrap(); + let winner = match (a, b) { + (Ok(()), Ok(())) if ignore_if_exists => "first", + (Ok(()), Err(Error::TableAlreadyExist { .. })) if !ignore_if_exists => "first", + (Err(Error::TableAlreadyExist { .. }), Ok(())) => "second", + results => panic!("unexpected creation results: {results:?}"), + }; + let table = first.get_table(&identifier).await.unwrap(); + assert_eq!(table.schema().fields()[0].name(), winner); + } + } + #[tokio::test] async fn test_create_table_rejects_an_unknown_type() { let (_temp_dir, catalog) = create_test_catalog(); diff --git a/crates/paimon/src/catalog/mod.rs b/crates/paimon/src/catalog/mod.rs index 72c936e33..f96f9f064 100644 --- a/crates/paimon/src/catalog/mod.rs +++ b/crates/paimon/src/catalog/mod.rs @@ -415,6 +415,17 @@ pub trait Catalog: Send + Sync { // ======================= table methods =============================== + /// Check that a base table exists. + /// + /// The default preserves [`Catalog::load_table`] lookup behavior. Override + /// with a metadata lookup to avoid initializing table storage. + /// + /// # Errors + /// Returns [`Error::TableNotExist`] for an absent table and propagates lookup errors. + async fn check_table_exists(&self, identifier: &Identifier) -> Result<()> { + self.load_table(identifier).await.map(|_| ()) + } + /// Get table metadata for the given identifier. /// /// # Errors @@ -481,6 +492,19 @@ pub trait Catalog: Send + Sync { ignore_if_exists: bool, ) -> Result<()>; + /// Create a table for immediate population. Only one concurrent creator may + /// succeed. Return [`Error::TableAlreadyExist`] if the target exists. + /// The default implementation returns [`Error::Unsupported`]. + async fn create_table_exclusive( + &self, + _identifier: &Identifier, + _creation: Schema, + ) -> Result<()> { + Err(Error::Unsupported { + message: "This catalog does not support exclusive table creation".to_string(), + }) + } + /// Drop a table. System tables cannot be dropped. /// /// # Errors diff --git a/crates/paimon/src/catalog/rest/rest_catalog.rs b/crates/paimon/src/catalog/rest/rest_catalog.rs index 04513355c..82f0d7efe 100644 --- a/crates/paimon/src/catalog/rest/rest_catalog.rs +++ b/crates/paimon/src/catalog/rest/rest_catalog.rs @@ -29,7 +29,7 @@ use crate::api::rest_api::RESTApi; use crate::api::rest_error::RestError; use crate::api::{ DataPolicy, GetTagResponse, ListPermissionsRequest, ListPoliciesRequest, PagedList, - PermissionAssignment, PermissionResource, PolicyType, + PermissionAssignment, PermissionResource, PolicyType, ResourceType, }; use crate::catalog::{ list_partitions_from_file_system, Catalog, Database, Identifier, DB_LOCATION_PROP, @@ -279,6 +279,12 @@ impl Catalog for RESTCatalog { // ======================= table methods =============================== + async fn check_table_exists(&self, identifier: &Identifier) -> Result<()> { + RESTEnv::fetch_table_response(identifier, &self.api) + .await + .map(|_| ()) + } + async fn get_table(&self, identifier: &Identifier) -> Result
{ RESTEnv::load_table( identifier, @@ -351,12 +357,20 @@ impl Catalog for RESTCatalog { .api .create_table(identifier, creation) .await - .map_err(|e| map_rest_error_for_table(e, identifier)); + .map_err(|e| map_table_metadata_error(e, identifier, ResourceType::Database)); ignore_error_if(result, |e| { ignore_if_exists && matches!(e, Error::TableAlreadyExist { .. }) }) } + async fn create_table_exclusive( + &self, + identifier: &Identifier, + creation: Schema, + ) -> Result<()> { + self.create_table(identifier, creation, false).await + } + async fn drop_table(&self, identifier: &Identifier, ignore_if_not_exists: bool) -> Result<()> { let result = self .api @@ -921,6 +935,32 @@ fn map_rest_error_for_table(err: Error, identifier: &Identifier) -> Error { } } +/// An untyped 404 refers to the table on GET and to its database on CREATE. +pub(crate) fn map_table_metadata_error( + error: Error, + identifier: &Identifier, + missing_resource: ResourceType, +) -> Error { + let resource = match &error { + Error::RestApi { + source: RestError::NoSuchResource { resource_type, .. }, + } => resource_type + .as_deref() + .unwrap_or(missing_resource.as_str()), + Error::RestApi { + source: RestError::AlreadyExists { resource_type, .. }, + } => resource_type.as_deref().unwrap_or("table"), + _ => return error, + }; + if resource.eq_ignore_ascii_case("database") { + map_rest_error_for_database(error, identifier.database()) + } else if resource.eq_ignore_ascii_case("table") || resource.eq_ignore_ascii_case("column") { + map_rest_error_for_table(error, identifier) + } else { + error + } +} + fn map_table_not_exist(error: Error, identifier: &Identifier) -> Error { match error { Error::RestApi { @@ -1458,7 +1498,7 @@ mod tests { } #[test] - fn test_column_error_mapping() { + fn test_table_error_mapping_preserves_the_resource_type() { let identifier = Identifier::new("db", "table"); let not_exist = Error::RestApi { @@ -1491,9 +1531,7 @@ mod tests { other => panic!("expected ColumnAlreadyExist, got {other:?}"), } - // A table-level 404 (no `column` resource type) must still map to - // `TableNotExist`, not a column error. - let table_missing = Error::RestApi { + let untyped_missing = || Error::RestApi { source: RestError::NoSuchResource { resource_type: None, resource_name: None, @@ -1501,8 +1539,25 @@ mod tests { }, }; assert!(matches!( - map_rest_error_for_table(table_missing, &identifier), - Error::TableNotExist { .. } + map_table_metadata_error(untyped_missing(), &identifier, ResourceType::Table), + Error::TableNotExist { full_name } if full_name == identifier.full_name() + )); + assert!(matches!( + map_table_metadata_error(untyped_missing(), &identifier, ResourceType::Database), + Error::DatabaseNotExist { database } if database == identifier.database() + )); + let policy_conflict = Error::RestApi { + source: RestError::AlreadyExists { + resource_type: Some("POLICY".into()), + resource_name: Some("readers".into()), + message: "policy already exists".into(), + }, + }; + assert!(matches!( + map_table_metadata_error(policy_conflict, &identifier, ResourceType::Database), + Error::RestApi { + source: RestError::AlreadyExists { .. } + } )); } } diff --git a/crates/paimon/src/io/file_io.rs b/crates/paimon/src/io/file_io.rs index edc472a10..1c5c1ea76 100644 --- a/crates/paimon/src/io/file_io.rs +++ b/crates/paimon/src/io/file_io.rs @@ -118,6 +118,14 @@ enum FileIOBackend { Provider(Arc), } +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum AtomicPublication { + Required, + /// Fall back to a plain write only if storage lacks a conditional operation. + /// A failed conditional publication may already have taken effect. + IfSupported, +} + #[derive(Clone)] pub struct FileIO { backend: FileIOBackend, @@ -874,6 +882,10 @@ impl FileIO { self.file_format_metadata_cache .invalidate_row_sidecar_size(cache_path); } + self.invalidate_cached_blocks(path, cache_path).await + } + + async fn invalidate_cached_blocks(&self, path: &str, cache_path: Option) -> Result<()> { if let (Some(cache), Some(cache_path)) = ( self.cache.as_ref().filter(|cache| cache.is_cacheable(path)), cache_path, @@ -903,14 +915,31 @@ impl FileIO { format!("Failed to delete directory '{path}'"), ) })?; - if let Some(ref cache_path) = cache_path { + self.invalidate_cached_prefix(cache_path.as_deref()).await + } + + pub(crate) async fn invalidate_directory_cache(&self, path: &str) -> Result<()> { + if self.cache.is_none() + && self + .file_format_metadata_cache + .row_sidecar_sizes + .get() + .is_none() + { + return Ok(()); + } + let (_, _, cache_path) = self.create_with_cache_path(path).await?; + self.invalidate_cached_prefix(cache_path.as_deref()).await + } + + async fn invalidate_cached_prefix(&self, cache_path: Option<&str>) -> Result<()> { + if let Some(cache_path) = cache_path { self.file_format_metadata_cache .invalidate_row_sidecar_prefix(cache_path); + if let Some(cache) = &self.cache { + cache.invalidate_prefix(cache_path).await?; + } } - if let (Some(cache), Some(cache_path)) = (&self.cache, cache_path) { - cache.invalidate_prefix(&cache_path).await?; - } - Ok(()) } @@ -948,6 +977,16 @@ impl FileIO { &self, path: &str, bytes: Bytes, + ) -> Result { + self.write_new_file(path, bytes, AtomicPublication::Required) + .await + } + + pub(crate) async fn write_new_file( + &self, + path: &str, + bytes: Bytes, + publication: AtomicPublication, ) -> Result { if let FileIOBackend::Provider(provider) = &self.backend { if let Some(published) = provider @@ -958,7 +997,7 @@ impl FileIO { } } let source = self.file_source(path)?; - let (op, relative_path, _) = source.resolve(path).await?; + let (op, relative_path, cache_path) = source.resolve(path).await?; #[cfg(feature = "storage-fs")] if matches!(source, FileSource::Static { local_fs: true, .. }) { @@ -969,18 +1008,26 @@ impl FileIO { let capability = info.capability(); let result = if capability.rename_with_if_not_exists { let staging_path = format!("{relative_path}.tmp-{}", uuid::Uuid::new_v4()); - op.write(&staging_path, bytes).await?; + let discard_staging = || async { + if let Err(error) = op.delete(&staging_path).await { + log::warn!( + "Failed to delete staging file '{staging_path}' for '{path}': {error}" + ); + } + }; + if let Err(error) = op.write(&staging_path, bytes).await { + discard_staging().await; + return Err(Error::from_opendal_with_context( + error, + format!("Failed to stage '{path}'"), + )); + } let result = op .rename_with(&staging_path, &relative_path) .if_not_exists(true) .await; - if result.as_ref().is_err_and(|error| { - matches!( - error.kind(), - opendal::ErrorKind::AlreadyExists | opendal::ErrorKind::ConditionNotMatch - ) - }) { - let _ = op.delete(&staging_path).await; + if result.is_err() { + discard_staging().await; } result } else if capability.write_with_if_not_exists @@ -994,10 +1041,18 @@ impl FileIO { .if_not_exists(true) .await .map(|_| ()) + } else if publication == AtomicPublication::IfSupported { + op.write(&relative_path, bytes).await.map_err(|error| { + Error::from_opendal_with_context(error, format!("Failed to write '{path}'")) + })?; + self.invalidate_cached_blocks(path, cache_path).await?; + return Ok(true); } else { return Err(Error::IoUnsupported { message: format!( - "Storage '{}' cannot atomically publish a complete snapshot without overwriting '{path}'; use a catalog SnapshotCommit with concurrency control", + "Storage '{}' cannot atomically publish '{path}' without overwriting it. \ + Use storage or a FileIOProvider with atomic publication support. \ + For snapshots, a catalog SnapshotCommit can also coordinate publication.", info.scheme() ), }); @@ -1028,14 +1083,14 @@ impl FileIO { ) -> Result { // OpenDAL's filesystem conditional write exposes the target before // its contents are complete. Stage locally, then publish without - // replacing another process's snapshot. + // replacing a file published by another process. let relative = normalize_path(relative_path); if std::path::Path::new(&relative) .components() .any(|component| matches!(component, std::path::Component::ParentDir)) { return Err(Error::IoUnsupported { - message: format!("Cannot publish a snapshot through a parent path: '{path}'"), + message: format!("Cannot atomically publish '{path}' through a parent path"), }); } let target = std::path::Path::new(&op.info().root()).join(relative); diff --git a/crates/paimon/src/table/data_file_writer.rs b/crates/paimon/src/table/data_file_writer.rs index a97ebea26..f920c4b7c 100644 --- a/crates/paimon/src/table/data_file_writer.rs +++ b/crates/paimon/src/table/data_file_writer.rs @@ -573,7 +573,9 @@ impl DataFileWriter { while self.in_flight_closes.join_next().await.is_some() {} for path in self.created_paths.drain(..) { if self.delete_file_upon_abort { - let _ = self.file_io.delete_file(&path).await; + if let Err(error) = self.file_io.delete_file(&path).await { + log::warn!("Failed to discard uncommitted file '{path}': {error}"); + } } } self.written_files.clear(); @@ -585,7 +587,9 @@ impl DataFileWriter { } for file in files { for path in file.collect_files(self.bucket_dir()) { - let _ = self.file_io.delete_file(&path).await; + if let Err(error) = self.file_io.delete_file(&path).await { + log::warn!("Failed to discard uncommitted file '{path}': {error}"); + } } } } diff --git a/crates/paimon/src/table/rest_env.rs b/crates/paimon/src/table/rest_env.rs index bec6e0a48..a68bdc77c 100644 --- a/crates/paimon/src/table/rest_env.rs +++ b/crates/paimon/src/table/rest_env.rs @@ -19,9 +19,10 @@ use crate::api::rest_api::RESTApi; use crate::api::rest_error::RestError; -use crate::api::GetTableResponse; +use crate::api::{GetTableResponse, ResourceType}; use crate::catalog::{ - FusePathResolver, Identifier, RESTToken, RESTTokenFileIO, DEFAULT_MAIN_BRANCH, + map_table_metadata_error, FusePathResolver, Identifier, RESTToken, RESTTokenFileIO, + DEFAULT_MAIN_BRANCH, }; use crate::common::{CatalogOptions, Options}; use crate::error::Error; @@ -330,7 +331,7 @@ impl RESTEnv { ) -> Result { api.get_table(identifier) .await - .map_err(|e| map_rest_error_for_table(e, identifier)) + .map_err(|e| map_table_metadata_error(e, identifier, ResourceType::Table)) } #[allow(clippy::too_many_arguments)] diff --git a/crates/paimon/src/table/table_commit.rs b/crates/paimon/src/table/table_commit.rs index 47ce9ba7e..a9bcd97fa 100644 --- a/crates/paimon/src/table/table_commit.rs +++ b/crates/paimon/src/table/table_commit.rs @@ -1099,7 +1099,9 @@ impl TableCommit { .abort(commit_messages) .await; } - self.delete_new_files(commit_messages).await?; + for failure in self.delete_new_files(commit_messages).await? { + log::warn!("Failed to discard uncommitted file: {failure}"); + } Ok(()) } diff --git a/crates/paimon/tests/mock_server.rs b/crates/paimon/tests/mock_server.rs index 6503331bf..6b7954c2b 100644 --- a/crates/paimon/tests/mock_server.rs +++ b/crates/paimon/tests/mock_server.rs @@ -36,13 +36,14 @@ use tokio::task::JoinHandle; use paimon::api::{ AlterDatabaseRequest, AlterTableRequest, AuditRESTResponse, AuthTableQueryResponse, ConfigResponse, CreateBranchRequest, CreateFunctionRequest, CreatePartitionsRequest, - CreateTagRequest, CreateViewRequest, DataPolicy, DropPartitionsRequest, DropPolicyRequest, - ErrorResponse, GetDatabaseResponse, GetFunctionResponse, GetTableResponse, GetTagResponse, - GetViewResponse, ListBranchesResponse, ListDatabasesResponse, ListFunctionsResponse, - ListPartitionsByFilterRequest, ListPartitionsByNamesRequest, ListPartitionsResponse, - ListPermissionsResponse, ListPoliciesResponse, ListTablesResponse, ListViewsResponse, - PermissionAssignment, PermissionResource, PolicyRequest, PolicyType, RESTUtil, - RenameTableRequest, ResourcePaths, ResourceType, RevokePermissionRequest, RollbackTableRequest, + CreateTableRequest, CreateTagRequest, CreateViewRequest, DataPolicy, DropPartitionsRequest, + DropPolicyRequest, ErrorResponse, GetDatabaseResponse, GetFunctionResponse, GetTableResponse, + GetTagResponse, GetViewResponse, ListBranchesResponse, ListDatabasesResponse, + ListFunctionsResponse, ListPartitionsByFilterRequest, ListPartitionsByNamesRequest, + ListPartitionsResponse, ListPermissionsResponse, ListPoliciesResponse, ListTablesResponse, + ListViewsResponse, PermissionAssignment, PermissionResource, PolicyRequest, PolicyType, + RESTUtil, RenameTableRequest, ResourcePaths, ResourceType, RevokePermissionRequest, + RollbackTableRequest, }; use paimon::catalog::{Function, Identifier}; use paimon::spec::{CommitKind, Partition, Snapshot}; @@ -220,7 +221,7 @@ fn paginate( #[derive(Clone)] pub struct RESTServer { warehouse: String, - _data_path: String, + data_path: String, config: ConfigResponse, inner: Arc>, resource_paths: ResourcePaths, @@ -233,7 +234,7 @@ impl RESTServer { /// Create a new RESTServer with initial databases. pub fn new( warehouse: String, - _data_path: String, + data_path: String, config: ConfigResponse, initial_dbs: Vec, ) -> Self { @@ -255,7 +256,7 @@ impl RESTServer { .collect(); RESTServer { - _data_path, + data_path, config, warehouse, inner: Arc::new(Mutex::new(MockState { @@ -784,26 +785,14 @@ impl RESTServer { Extension(state): Extension>, Json(payload): Json, ) -> impl IntoResponse { - // Extract table name from payload - let table_name = payload - .get("identifier") - .and_then(|id| id.get("object")) - .and_then(|o| o.as_str()) - .map(|s| s.to_string()); - - let table_name = match table_name { - Some(name) => name, - None => { - let err = ErrorResponse::new( - None, - None, - Some("Missing table name in identifier".to_string()), - Some(400), - ); + let request = match serde_json::from_value::(payload) { + Ok(request) => request, + Err(error) => { + let err = ErrorResponse::new(None, None, Some(error.to_string()), Some(400)); return (StatusCode::BAD_REQUEST, Json(err)).into_response(); } }; - + let table_name = request.identifier.object().to_string(); let mut s = state.inner.lock().unwrap(); // Check database exists @@ -828,15 +817,15 @@ impl RESTServer { return (StatusCode::CONFLICT, Json(err)).into_response(); } - // Create table response + let path = format!("{}/{db}.db/{table_name}", state.data_path); let response = GetTableResponse::new( - Some(table_name.clone()), + Some(uuid::Uuid::new_v4().to_string()), Some(db.clone()), Some(table_name), - None, + Some(path), Some(true), - None, - None, + Some(0), + Some(request.schema), AuditRESTResponse::new(None, None, None, None, None), ); s.tables.insert(key, response); diff --git a/crates/paimon/tests/rest_catalog_test.rs b/crates/paimon/tests/rest_catalog_test.rs index 3be7e7456..022fa26cc 100644 --- a/crates/paimon/tests/rest_catalog_test.rs +++ b/crates/paimon/tests/rest_catalog_test.rs @@ -1923,52 +1923,37 @@ async fn test_catalog_get_table_propagates_oss_options_in_else_branch() { #[tokio::test] async fn test_catalog_create_table() { let ctx = setup_catalog(vec!["default"]).await; - - let schema = test_schema(); let identifier = Identifier::new("default", "new_table"); - - let result = ctx.catalog.create_table(&identifier, schema, false).await; - assert!(result.is_ok(), "failed to create table: {result:?}"); - - // Verify table exists - let tables = ctx.catalog.list_tables("default").await.unwrap(); - assert!(tables.contains(&"new_table".to_string())); -} - -#[tokio::test] -async fn test_catalog_create_table_already_exists() { - let ctx = setup_catalog(vec!["default"]).await; - - // Add a table first - ctx.server.add_table("default", "existing_table"); - - let schema = test_schema(); - let identifier = Identifier::new("default", "existing_table"); - - // Create with ignore_if_exists=false should fail - let result = ctx.catalog.create_table(&identifier, schema, false).await; - assert!( - result.is_err(), - "creating duplicate table should fail when ignore_if_exists=false" - ); -} - -#[tokio::test] -async fn test_catalog_create_table_ignore_if_exists() { - let ctx = setup_catalog(vec!["default"]).await; - - // Add a table first - ctx.server.add_table("default", "existing_table"); - - let schema = test_schema(); - let identifier = Identifier::new("default", "existing_table"); - - // Create with ignore_if_exists=true should succeed - let result = ctx.catalog.create_table(&identifier, schema, true).await; - assert!( - result.is_ok(), - "creating duplicate table should succeed when ignore_if_exists=true" + ctx.catalog + .create_table(&identifier, test_schema(), false) + .await + .unwrap(); + for result in [ + ctx.catalog + .create_table(&identifier, test_schema(), false) + .await, + ctx.catalog + .create_table_exclusive(&identifier, test_schema()) + .await, + ] { + assert!(matches!( + result, + Err(paimon::Error::TableAlreadyExist { full_name }) if full_name == identifier.full_name() + )); + } + ctx.catalog + .create_table(&identifier, test_schema(), true) + .await + .unwrap(); + assert_eq!( + ctx.catalog.list_tables("default").await.unwrap(), + ["new_table"] ); + let missing = Identifier::new("missing_db", "new_table"); + assert!(matches!( + ctx.catalog.create_table_exclusive(&missing, test_schema()).await, + Err(paimon::Error::DatabaseNotExist { database }) if database == missing.database() + )); } #[tokio::test] diff --git a/docs/src/sql.md b/docs/src/sql.md index 7bb705b43..02ff96e2a 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -43,7 +43,7 @@ SQL support has two layers: - DataFusion provides the parser, query planner, optimizer, execution engine, expressions, scalar functions, aggregate functions, and window functions. SQL statements that `SQLContext` does not intercept are delegated to DataFusion. This includes the DataFusion SQL surface for `SELECT` queries, CTEs (including recursive CTEs), subqueries, joins including `LATERAL` joins, SQL lambda functions, grouping, `HAVING`, window clauses, `QUALIFY`, set operations, `ORDER BY`, `LIMIT`/`OFFSET`, `EXPLAIN`, information-schema commands such as `SHOW TABLES`, `DESCRIBE`, `COPY`, and ordinary `INSERT`. - Paimon-specific table management and row-level writes are implemented by `SQLContext`. This includes Paimon `CREATE TABLE`, `ALTER TABLE`, `DROP TABLE`, `CREATE TEMPORARY TABLE`, `CREATE TEMPORARY VIEW`, REST Catalog persistent `CREATE VIEW`, `DROP VIEW`, `CREATE FUNCTION`, and `DROP FUNCTION`, `DROP TEMPORARY TABLE` / `VIEW`, `INSERT OVERWRITE ... PARTITION`, `UPDATE`, `DELETE`, `TRUNCATE TABLE`, `ALTER TABLE ... ADD PARTITION`, `ALTER TABLE ... DROP PARTITION`, `SHOW PARTITIONS`, `MSCK REPAIR TABLE`, `ANALYZE TABLE`, `CALL sys.*`, Paimon time travel, and `SET` / `RESET 'paimon.*'`. `MERGE INTO` uses DataFusion's native planner and the Paimon table provider for execution. -Not every DataFusion DDL/DML statement maps to a Paimon table operation. For Paimon catalogs, `CREATE EXTERNAL TABLE`, `LOCATION`, `CREATE MATERIALIZED VIEW`, and persistent `CREATE TABLE AS SELECT` are rejected or not implemented. Persistent `CREATE FUNCTION` is supported only for the REST Catalog SQL scalar form documented below. DataFusion `COPY` can export query results to files; it does not create or commit Paimon table files. +Not every DataFusion DDL/DML statement maps to a Paimon table operation. For Paimon catalogs, `CREATE EXTERNAL TABLE`, `LOCATION`, `CREATE MATERIALIZED VIEW`, and `CREATE OR REPLACE TABLE ... AS SELECT` are rejected or not implemented. Persistent `CREATE FUNCTION` is supported only for the REST Catalog SQL scalar form documented below. DataFusion `COPY` can export query results to files; it does not create or commit Paimon table files. For the exact delegated SQL grammar, see the [DataFusion SQL Reference](https://datafusion.apache.org/user-guide/sql/index.html). @@ -913,7 +913,76 @@ mixed-case or uppercase field when normalization is enabled. Unsupported syntax (will return an error): - `CREATE EXTERNAL TABLE` - `LOCATION` -- `CREATE TABLE AS SELECT` + +### CREATE TABLE AS SELECT + +Use a read-only query to create and populate a persistent Paimon table of the +default type (`'type' = 'table'`). + +```sql +CREATE TABLE paimon.my_db.query_result +PARTITIONED BY (region) +WITH ('bucket' = '-1') +AS SELECT id, region FROM source WHERE id > 10; +``` + +Column names, types and nullability come from the analyzed query, even when it +returns zero rows. Use aliases and casts to choose output names and types. +Geospatial outputs preserve their geometry or geography type, CRS and edge algorithm. +Table options, comments and partition keys use the same validation as +`CREATE TABLE`. Place `PARTITIONED BY` before `AS` and list unqualified output +column names. Quote a name containing a dot, as in `PARTITIONED BY ("event.day")`. +CTAS rejects explicit column declarations and `OR REPLACE`. The read-only +requirement also applies inside CTEs and set operations. CTAS accepts one +table-options clause (`WITH`, `TBLPROPERTIES` or `OPTIONS`) and rejects +modifiers such as `AUTO_INCREMENT`. + +Paimon stores `TIME` values with millisecond precision. CTAS accepts Arrow +second and millisecond time outputs, but rejects microsecond and nanosecond time +outputs before creating a table, including nested and dictionary-encoded outputs. +This also applies to empty results. DataFusion's `CAST(... AS TIME)` produces a +nanosecond time type. Cast it to `VARCHAR` to retain finer precision as text. + +Unqualified source tables and REST catalog SQL functions resolve in the +**session catalog and database at the start of the statement**, even for a +destination in another namespace. The same query plan supplies the schema and +the rows. Source and destination names follow the session's identifier +normalization setting. +Quotes preserve case and treat a name such as `"events.archive"` as one +identifier. Use `VERSION AS OF` with a snapshot ID or tag, or +`TIMESTAMP AS OF` with a timestamp, to select a source version. Tables in a join +can read different snapshots. + +`IF NOT EXISTS` on an existing target skips source planning and execution, even +when that target uses an external table engine. Filesystem and REST catalogs +check only target metadata, so an Object Table can be skipped even when its +storage is unavailable to the client. A missing database, a permission +failure or a transport error ends the statement. If another client creates the +target first, CTAS returns a table-already-exists error. With `IF NOT EXISTS` +it succeeds without writing. + +Filesystem CTAS requires all concurrent creators to publish the initial schema +atomically without overwriting it. On OSS, the bucket must never have had +versioning enabled. Ordinary `CREATE TABLE` uses conditional publication when +available and propagates publication errors. On storage that lacks this +capability, such as the HDFS-native backend, ordinary `CREATE TABLE` can +register a Format Table with a plain write. Coordinate concurrent ordinary +creates of the same name on such storage. + +A lost create response can leave the table visible. After confirmed creation, +CTAS loads the table for writing. A load error identifies the newly created +table and returns before source execution. As with an ordinary INSERT, +coordinate DROP, RENAME and table replacement with active writers. + +**A population failure retains the created table.** Other clients can write to +it as soon as creation succeeds. Before commit, the writer attempts to delete +its outstanding files after a failure or caller cancellation. Check cleanup +warnings for paths that may need attention. An in-progress write finishes +before cleanup. If the caller cancels after commit starts, publication +continues and rows may become visible. A commit error leaves prepared files +in place because a published snapshot may reference them. Process termination +can interrupt cleanup. Inspect the table and its snapshots before retrying. +`IF NOT EXISTS` skips a retained table. ### Partitioned Tables