diff --git a/Cargo.lock b/Cargo.lock index e3ec5e217..f0869b901 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5081,6 +5081,7 @@ dependencies = [ "tar", "tempfile", "tokio", + "uuid", ] [[package]] diff --git a/crates/integration_tests/Cargo.toml b/crates/integration_tests/Cargo.toml index 75ec66175..b5e68ef8f 100644 --- a/crates/integration_tests/Cargo.toml +++ b/crates/integration_tests/Cargo.toml @@ -38,3 +38,4 @@ tar = "0.4" tempfile = "3" serde_json = "1" axum = { version = "0.7", features = ["macros", "tokio", "http1", "http2"] } +uuid = { version = "1", features = ["v4"] } diff --git a/crates/integrations/datafusion/README.md b/crates/integrations/datafusion/README.md index a91a968ab..c12d7e8dc 100644 --- a/crates/integrations/datafusion/README.md +++ b/crates/integrations/datafusion/README.md @@ -149,4 +149,21 @@ let view = ctx.sql("SELECT * FROM analytics_view").await?; let function = ctx.sql("SELECT plus_one(score) FROM scores").await?; ``` +## Persistent CREATE TABLE AS SELECT + +Use `CREATE TABLE AS SELECT` to materialize a query as a native Paimon table. +The query determines the destination schema. In this example, `scores` resolves +in the session's current catalog and database. + +```sql +CREATE TABLE paimon.reporting.results +WITH ('bucket' = '-1') +AS SELECT id, score FROM scores WHERE score > 10; +``` + +`IF NOT EXISTS` skips an existing target. If population fails after creation, +the destination remains in the catalog. For supported types, concurrent DDL, +cancellation and recovery after a commit error, see the +[SQL reference](../../../docs/src/sql.md#create-table-as-select). + See the [documentation](https://paimon.apache.org/docs/rust/datafusion/) for getting started guide and more details. diff --git a/crates/integrations/datafusion/src/format_partition_analyze.rs b/crates/integrations/datafusion/src/format_partition_analyze.rs index 01946d70c..99e97de3a 100644 --- a/crates/integrations/datafusion/src/format_partition_analyze.rs +++ b/crates/integrations/datafusion/src/format_partition_analyze.rs @@ -57,7 +57,7 @@ pub(crate) async fn execute_analyze( "ANALYZE TABLE requires COMPUTE STATISTICS".to_string(), )); } - SQLContext::ensure_partition_command_target(table_name, "ANALYZE TABLE")?; + SQLContext::ensure_main_branch_base_table(table_name, "ANALYZE TABLE")?; ctx.reject_temporary_write_target(table_name, "ANALYZE TABLE") .await?; let (catalog, _catalog_name, identifier) = ctx.resolve_catalog_and_table(table_name)?; diff --git a/crates/integrations/datafusion/src/format_partition_ddl.rs b/crates/integrations/datafusion/src/format_partition_ddl.rs index c2d4eeac9..33d39f587 100644 --- a/crates/integrations/datafusion/src/format_partition_ddl.rs +++ b/crates/integrations/datafusion/src/format_partition_ddl.rs @@ -112,7 +112,7 @@ pub(crate) async fn execute_show_partitions( show_partitions: &ShowPartitionsStatement, enable_ident_normalization: bool, ) -> DFResult { - SQLContext::ensure_partition_command_target(&show_partitions.table_name, "SHOW PARTITIONS")?; + SQLContext::ensure_main_branch_base_table(&show_partitions.table_name, "SHOW PARTITIONS")?; let (catalog, _catalog_name, identifier) = ctx.resolve_catalog_and_table(&show_partitions.table_name)?; let table = catalog diff --git a/crates/integrations/datafusion/src/format_partition_repair.rs b/crates/integrations/datafusion/src/format_partition_repair.rs index 1f018b382..78dcf44c2 100644 --- a/crates/integrations/datafusion/src/format_partition_repair.rs +++ b/crates/integrations/datafusion/src/format_partition_repair.rs @@ -36,7 +36,7 @@ pub(crate) async fn execute_msck(ctx: &SQLContext, msck: &Msck) -> DFResult Result { + use DataType::*; + if data.child_data().is_empty() { + return Ok(data); + } + let children = data + .child_data() + .iter() + .cloned() + .map(normalize_map_ordering) + .collect::, _>>()?; + let field = |original: &FieldRef, index: usize| { + Arc::new( + original + .as_ref() + .clone() + .with_data_type(children[index].data_type().clone()), + ) + }; + let data_type = match data.data_type() { + Map(entries, _) => Map(field(entries, 0), false), + Struct(fields) => Struct( + fields + .iter() + .enumerate() + .map(|(i, f)| field(f, i)) + .collect(), + ), + List(f) => List(field(f, 0)), + LargeList(f) => LargeList(field(f, 0)), + FixedSizeList(f, size) => FixedSizeList(field(f, 0), *size), + Dictionary(key, _) => Dictionary(key.clone(), Box::new(children[0].data_type().clone())), + _ => return Ok(data), + }; + if &data_type == data.data_type() { + return Ok(data); + } + data.into_builder() + .data_type(data_type) + .child_data(children) + .build() +} + +fn cast_column(column: &ArrayRef, target: &DataType) -> Result { + if column.data_type() == target { + return Ok(Arc::clone(column)); + } + if !can_cast_types(column.data_type(), target) { + let data = normalize_map_ordering(column.to_data())?; + if data.data_type() != column.data_type() { + return cast(&make_array(data), target); + } + } + cast(column, target) +} + +pub(crate) fn cast_batch_to_schema( + batch: RecordBatch, + schema: &SchemaRef, +) -> DFResult { + if batch.schema_ref() == schema { + return Ok(batch); + } + if batch.num_columns() != schema.fields().len() { + return Err(DataFusionError::Execution(format!( + "Cannot cast a batch with {} columns to a schema with {} fields", + batch.num_columns(), + schema.fields().len() + ))); + } + let columns = batch + .columns() + .iter() + .zip(schema.fields()) + .map(|(column, field)| cast_column(column, field.data_type())) + .collect::, _>>()?; + let options = RecordBatchOptions::new().with_row_count(Some(batch.num_rows())); + RecordBatch::try_new_with_options(Arc::clone(schema), columns, &options).map_err(Into::into) +} diff --git a/crates/integrations/datafusion/src/physical_plan/scan.rs b/crates/integrations/datafusion/src/physical_plan/scan.rs index 0b60f8073..fce9e6ffa 100644 --- a/crates/integrations/datafusion/src/physical_plan/scan.rs +++ b/crates/integrations/datafusion/src/physical_plan/scan.rs @@ -18,11 +18,11 @@ use std::sync::Arc; use datafusion::arrow::array::BooleanArray; -use datafusion::arrow::compute::{cast, filter_record_batch}; +use datafusion::arrow::compute::filter_record_batch; use datafusion::arrow::datatypes::{ DataType as ArrowDataType, SchemaRef as ArrowSchemaRef, TimeUnit, }; -use datafusion::arrow::record_batch::{RecordBatch, RecordBatchOptions}; +use datafusion::arrow::record_batch::RecordBatch; use datafusion::common::stats::Precision; use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{ColumnStatistics, ScalarValue, Statistics}; @@ -64,33 +64,6 @@ use paimon::DataSplit; use crate::error::to_datafusion_error; use crate::filter_pushdown::{is_float_zero, scalar_to_datum}; -fn to_datafusion_batch(batch: RecordBatch, schema: &ArrowSchemaRef) -> DFResult { - if batch.num_columns() != schema.fields().len() { - return Err(datafusion::error::DataFusionError::Execution(format!( - "Paimon reader returned {} columns for DataFusion schema with {} fields", - batch.num_columns(), - schema.fields().len() - ))); - } - - let row_count = batch.num_rows(); - let columns = batch - .columns() - .iter() - .zip(schema.fields()) - .map(|(column, field)| { - if column.data_type() == field.data_type() { - Ok(Arc::clone(column)) - } else { - cast(column.as_ref(), field.data_type()).map_err(Into::into) - } - }) - .collect::>>()?; - let options = RecordBatchOptions::new().with_row_count(Some(row_count)); - - RecordBatch::try_new_with_options(Arc::clone(schema), columns, &options).map_err(Into::into) -} - #[derive(Debug)] struct DataFusionRowFilterFactory { predicate: Arc, @@ -1125,7 +1098,7 @@ impl PaimonTableScan { let stream = stream.map(move |result| { let mut batch = result .map_err(to_datafusion_error) - .and_then(|batch| to_datafusion_batch(batch, &batch_schema))?; + .and_then(|batch| super::cast_batch_to_schema(batch, &batch_schema))?; // The decoder hook is an optimization and may be unavailable // for a file/path. Retain every original live expression as // the exact fallback; evaluating it on decoder survivors is diff --git a/crates/integrations/datafusion/src/procedures/compat.rs b/crates/integrations/datafusion/src/procedures/compat.rs index d4569fff5..5eed29b48 100644 --- a/crates/integrations/datafusion/src/procedures/compat.rs +++ b/crates/integrations/datafusion/src/procedures/compat.rs @@ -58,7 +58,7 @@ async fn validate_write_target( Ident::with_quote('"', identifier.database()), Ident::with_quote('"', identifier.object()), ]); - SQLContext::ensure_partition_command_target(&name, operation)?; + SQLContext::ensure_main_branch_base_table(&name, operation)?; if let Some(sql_ctx) = sql_ctx { sql_ctx.ensure_no_time_travel_for_write(operation)?; sql_ctx diff --git a/crates/integrations/datafusion/src/relation_planner.rs b/crates/integrations/datafusion/src/relation_planner.rs index 10b2679db..4334d04d1 100644 --- a/crates/integrations/datafusion/src/relation_planner.rs +++ b/crates/integrations/datafusion/src/relation_planner.rs @@ -39,6 +39,26 @@ use crate::catalog::{ }; use crate::table::{ObjectTableProvider, PaimonTableProvider}; +#[derive(Debug)] +pub(crate) struct SnapshotRelationPlanner( + pub HashMap)>, +); + +impl RelationPlanner for SnapshotRelationPlanner { + fn plan_relation( + &self, + relation: TableFactor, + _: &mut dyn RelationPlannerContext, + ) -> DFResult { + if let TableFactor::Table { name, .. } = &relation { + if let Some((table, provider)) = self.0.get(&name.to_string()) { + return plan_scan(relation, table.clone(), provider.clone()); + } + } + Ok(RelationPlanning::Original(Box::new(relation))) + } +} + /// Resolves Paimon time travel, and registered engine snapshot IDs and timestamps. #[derive(Debug)] pub struct PaimonRelationPlanner; diff --git a/crates/integrations/datafusion/src/sql_context.rs b/crates/integrations/datafusion/src/sql_context.rs index a8a98b7d3..f4348765c 100644 --- a/crates/integrations/datafusion/src/sql_context.rs +++ b/crates/integrations/datafusion/src/sql_context.rs @@ -28,6 +28,7 @@ //! - `DROP DATABASE [IF EXISTS] [catalog.]db [CASCADE]` //! - `USE [catalog.]db` //! - `CREATE TABLE db.t (col TYPE, ..., PRIMARY KEY (col, ...)) [PARTITIONED BY (col, ...)] [WITH ('key' = 'val')]` +//! - `CREATE TABLE [IF NOT EXISTS] db.t [PARTITIONED BY (col, ...)] [WITH ('key' = 'val')] AS query` //! - `ALTER TABLE db.t ADD COLUMN col TYPE` //! - `ALTER TABLE db.t DROP COLUMN col` //! - `ALTER TABLE db.t RENAME COLUMN old TO new` @@ -78,7 +79,7 @@ use datafusion::sql::sqlparser::ast::{ use datafusion::sql::sqlparser::dialect::GenericDialect; use datafusion::sql::sqlparser::keywords::Keyword; use datafusion::sql::sqlparser::parser::Parser; -use datafusion::sql::sqlparser::tokenizer::{Token, Tokenizer}; +use datafusion::sql::sqlparser::tokenizer::{Location, Token, Tokenizer}; use datafusion_spark::SessionStateBuilderSpark; use futures::StreamExt; use paimon::catalog::{parse_object_name, Catalog, Identifier, ParsedObjectName}; @@ -94,6 +95,8 @@ use crate::error::to_datafusion_error; use crate::table::PaimonTableProvider; use crate::{BlobReaderRegistry, DynamicOptions}; +mod ctas; + /// A SQL context that supports registering multiple Paimon catalogs and executing SQL. /// /// # Example @@ -490,17 +493,16 @@ impl SQLContext { /// Execute a SQL statement with options for statements delegated to DataFusion. pub async fn sql_with_options(&self, sql: &str, options: SQLOptions) -> DFResult { - let is_create_table = looks_like_create_table(sql); + // Keep the dispatcher off callers' stacks while catalog and query futures are polled. + Box::pin(self.execute_statement(sql, options)).await + } + + async fn execute_statement(&self, sql: &str, options: SQLOptions) -> DFResult { let enable_ident_normalization = self.ctx.enable_ident_normalization(); - let (rewritten_sql, partition_keys) = if is_create_table { - extract_partition_by(sql, enable_ident_normalization)? - } else { - (sql.to_string(), vec![]) - }; - if contains_time_travel_keyword(&rewritten_sql) { - // Time-travel queries are not DDL; skip our own parsing and handle directly. - return self.handle_time_travel_query(&rewritten_sql, options).await; - } + let (mut rewritten_sql, partition_keys) = + extract_partition_by(sql, enable_ident_normalization)?; + let time_travel_sources = + mask_time_travel_sources(&mut rewritten_sql, enable_ident_normalization); if let Some(show_partitions) = crate::format_partition_ddl::parse_show_partitions(&rewritten_sql)? { @@ -526,33 +528,46 @@ impl SQLContext { )); } + if !time_travel_sources.is_empty() { + validate_time_travel_sources(&statements[0], &time_travel_sources)?; + } + if let Statement::CreateTable(ct) = &mut statements[0] { + if let Some(query) = ct.query.take_if(|_| !ct.temporary) { + return Box::pin(self.handle_create_table_as( + ct, + query, + partition_keys, + enable_ident_normalization, + options, + &time_travel_sources, + )) + .await; + } + } + if !time_travel_sources.is_empty() + || matches!( + statements[0], + Statement::Query(_) | Statement::Explain { .. } + ) + { + return Box::pin(self.plan_query(statements.remove(0), options, &time_travel_sources)) + .await; + } + let rewritten; let stored_definition = matches!(&statements[0], Statement::CreateView(view) if view.temporary || self.is_paimon_catalog_ref(&TableReference::from(view.name.to_string().as_str()))) || matches!(&statements[0], Statement::CreateFunction(_)); let changed = !stored_definition && { - let binary_changed = crate::binary_cast::rewrite(&mut statements[0])?; - let regex_changed = crate::regex_operators::rewrite(&mut statements[0]); - let lateral_changed = crate::lateral_views::rewrite(&mut statements[0])?; - let compat_changed = - crate::expression_compat::rewrite(&mut statements[0], &self.ctx.state())?; + let expressions_changed = self.rewrite_expressions(&mut statements[0])?; let state = self.ctx.state(); - // Queries and temporary view bodies resolve after SQL function expansion below. - let max_pt_changed = if matches!( - statements[0], - Statement::Query(_) | Statement::Explain { .. } - ) || matches!(&statements[0], Statement::CreateView(view) if view.temporary) - { - false - } else { - Box::pin(crate::partition_functions::rewrite( - &mut statements[0], - &state, - &self.current_catalog_name(), - &state.config_options().catalog.default_schema, - )) - .await? - }; - binary_changed || regex_changed || lateral_changed || compat_changed || max_pt_changed + let partitions_changed = Box::pin(crate::partition_functions::rewrite( + &mut statements[0], + &state, + &self.current_catalog_name(), + &state.config_options().catalog.default_schema, + )) + .await?; + expressions_changed || partitions_changed }; let sql = if changed || vector_changed { rewritten = statements[0].to_string(); @@ -936,127 +951,63 @@ impl SQLContext { )) .await } - Statement::Query(_) | Statement::Explain { .. } => { - let mut statement = statements[0].clone(); - if let Statement::Explain { - statement: inner, .. - } = &mut statement - { - if let Statement::Merge(merge) = inner.as_mut() { - *merge = self.normalize_merge_into(merge).await?; - } - } - let current_catalog = self.current_catalog_name(); - let current_database = self - .ctx - .state() - .config_options() - .catalog - .default_schema - .clone(); - let expanded = crate::sql_function::expand_statement( - &self.ctx.state(), - statement, - &self.catalogs, - ¤t_catalog, - ¤t_database, - ) - .await?; - crate::read_files::sql(&self.ctx, &expanded.to_string(), options).await - } _ => crate::read_files::sql(&self.ctx, sql, options).await, } } - /// Rewrite time-travel references to temporary snapshot providers. - /// The tracker removes registrations after planning. - async fn handle_time_travel_query( + fn rewrite_expressions(&self, statement: &mut Statement) -> DFResult { + let binary = crate::binary_cast::rewrite(statement)?; + let regex = crate::regex_operators::rewrite(statement); + let lateral = crate::lateral_views::rewrite(statement)?; + let compat = crate::expression_compat::rewrite(statement, &self.ctx.state())?; + Ok(binary || regex || lateral || compat) + } + + async fn plan_query( &self, - sql: &str, - sql_options: SQLOptions, + mut statement: Statement, + options: SQLOptions, + sources: &[TimeTravelSource], ) -> DFResult { - let mut tracker = crate::merge_into::TempTableTracker::new(self); - - let version_clauses = extract_all_version_as_of(sql); - let timestamp_clauses = extract_all_timestamp_as_of(sql); - - if version_clauses.is_empty() && timestamp_clauses.is_empty() { - return Err(DataFusionError::Plan( - "Failed to parse time-travel clause in SQL".to_string(), - )); - } - - let mut replacements: Vec<((usize, usize), String)> = Vec::new(); - - for info in &version_clauses { - let table_ref: datafusion::common::TableReference = info.table_name.as_str().into(); - let (catalog, catalog_name, identifier) = - self.resolve_table_name_from_ref(&table_ref)?; - let provider = self - .time_travel_provider( - &catalog, - &catalog_name, - &identifier, - TimeTravelClause::Version(info.version.clone()), - ) - .await?; - - let uuid_name = format!("__paimon_tt_{}", uuid::Uuid::new_v4().as_simple()); - self.register_temp_table(uuid_name.as_str(), provider)?; - tracker.register(&uuid_name); - replacements.push((info.clause_range, uuid_name)); + self.rewrite_expressions(&mut statement)?; + if let Statement::Explain { + statement: inner, .. + } = &mut statement + { + if let Statement::Merge(merge) = inner.as_mut() { + *merge = self.normalize_merge_into(merge).await?; + } } - - for info in ×tamp_clauses { - let table_ref: datafusion::common::TableReference = info.table_name.as_str().into(); + let mut registrations = crate::merge_into::TempTableTracker::new(self); + let mut bindings = HashMap::new(); + for source in sources { let (catalog, catalog_name, identifier) = - self.resolve_table_name_from_ref(&table_ref)?; - let millis = Self::parse_timestamp_to_millis(&info.timestamp)?; + self.resolve_table_name_from_ref(&source.table)?; let provider = self - .time_travel_provider( - &catalog, - &catalog_name, - &identifier, - TimeTravelClause::TimestampMillis(millis), - ) + .time_travel_provider(&catalog, &catalog_name, &identifier, &source.clause) .await?; - - let uuid_name = format!("__paimon_tt_{}", uuid::Uuid::new_v4().as_simple()); - self.register_temp_table(uuid_name.as_str(), provider)?; - tracker.register(&uuid_name); - replacements.push((info.clause_range, uuid_name)); + self.register_temp_table(source.binding.as_str(), provider.clone())?; + registrations.register(&source.binding); + bindings.insert(source.binding.clone(), (source.table.clone(), provider)); } - - // Replace from right to left to preserve byte offsets. - replacements.sort_by_key(|r| std::cmp::Reverse(r.0 .0)); - - let mut rewritten_sql = sql.to_string(); - for ((start, end), uuid_name) in &replacements { - rewritten_sql = format!( - "{}{}{}", - &rewritten_sql[..*start], - uuid_name, - &rewritten_sql[*end..] - ); - } - - let current_catalog = self.current_catalog_name(); - let current_database = self - .ctx - .state() - .config_options() - .catalog - .default_schema - .clone(); - let expanded = crate::sql_function::expand_sql( - &self.ctx.state(), - &rewritten_sql, + let state = self.ctx.state(); + let expanded = crate::sql_function::expand_statement( + &state, + statement, &self.catalogs, - ¤t_catalog, - ¤t_database, + &self.current_catalog_name(), + &state.config_options().catalog.default_schema, ) .await?; - crate::read_files::sql(&self.ctx, &expanded, sql_options).await + let sql = expanded.to_string(); + if sources.is_empty() { + return crate::read_files::sql(&self.ctx, &sql, options).await; + } + let context = SessionContext::new_with_state(state); + context.register_relation_planner(Arc::new( + crate::relation_planner::SnapshotRelationPlanner(bindings), + ))?; + crate::read_files::sql(&context, &sql, options).await } async fn time_travel_provider( @@ -1064,7 +1015,7 @@ impl SQLContext { catalog: &Arc, catalog_name: &str, identifier: &Identifier, - clause: TimeTravelClause, + clause: &TimeTravelClause, ) -> DFResult> { use crate::catalog::EngineTimeTravel; use paimon::spec::{SCAN_TIMESTAMP_MILLIS_OPTION, SCAN_VERSION_OPTION}; @@ -1096,10 +1047,10 @@ impl SQLContext { } let travel = match clause { TimeTravelClause::Version(version) => { - EngineTimeTravel::Snapshot(crate::catalog::parse_engine_snapshot_id(&version)?) + EngineTimeTravel::Snapshot(crate::catalog::parse_engine_snapshot_id(version)?) } - TimeTravelClause::TimestampMillis(millis) => { - EngineTimeTravel::TimestampMillis(millis) + TimeTravelClause::Timestamp(timestamp) => { + EngineTimeTravel::TimestampMillis(Self::parse_timestamp_to_millis(timestamp)?) } }; let provider = self @@ -1126,10 +1077,11 @@ impl SQLContext { let (table, base, system_name) = crate::table_loader::loaded_table_for_read(loaded, identifier).await?; let (option, value) = match clause { - TimeTravelClause::Version(version) => (SCAN_VERSION_OPTION, version), - TimeTravelClause::TimestampMillis(millis) => { - (SCAN_TIMESTAMP_MILLIS_OPTION, millis.to_string()) - } + TimeTravelClause::Version(version) => (SCAN_VERSION_OPTION, version.clone()), + TimeTravelClause::Timestamp(timestamp) => ( + SCAN_TIMESTAMP_MILLIS_OPTION, + Self::parse_timestamp_to_millis(timestamp)?.to_string(), + ), }; let mut options = self.dynamic_options.read().unwrap().clone(); options.insert(option.to_string(), value); @@ -1236,73 +1188,14 @@ impl SQLContext { "LOCATION is not supported for Paimon tables. Table path is determined by the catalog warehouse.".to_string(), )); } - if ct.query.is_some() { - return Err(DataFusionError::Plan( - "CREATE TABLE AS SELECT is not yet supported for Paimon tables.".to_string(), - )); - } - let identifier = self.resolve_table_name(&ct.name)?; - - let mut builder = paimon::spec::Schema::builder(); - let mut comments = Vec::new(); - if let Some(comment) = &ct.comment { - comments.push(comment); - } - let options = match &ct.table_options { - CreateTableOptions::With(options) - | CreateTableOptions::Options(options) - | CreateTableOptions::TableProperties(options) - | CreateTableOptions::Plain(options) => options.as_slice(), - CreateTableOptions::None => &[], - }; - for option in options { - if let SqlOption::Comment(comment) = option { - comments.push(comment); - } - } - if comments.len() > 1 { - return Err(DataFusionError::Plan("Duplicate table COMMENT".into())); - } - if let Some(CommentDef::WithEq(comment) | CommentDef::WithoutEq(comment)) = - comments.first().copied() - { - builder = builder.comment(Some(comment.clone())); - } - let table_options = extract_options(&ct.table_options)?; - let column_names: Vec = ct - .columns - .iter() - .map(|column| normalize_schema_identifier(&column.name, enable_ident_normalization)) - .collect(); - - // Columns - for (col, column_name) in ct.columns.iter().zip(&column_names) { - let paimon_type = column_def_to_paimon_type(col)?; - let comment = column_def_comment(col); - builder = builder.column_with_description(column_name.clone(), paimon_type, comment); - } - - builder = builder.primary_key(extract_primary_keys(ct, enable_ident_normalization)?); - - // Partition keys (extracted and validated before parsing) - if !partition_keys.is_empty() { - for pk in &partition_keys { - if !column_names.contains(pk) { - return Err(DataFusionError::Plan(format!( - "PARTITIONED BY column '{pk}' is not defined in the table" - ))); - } - } - builder = builder.partition_keys(partition_keys); - } - - // Table options from WITH ('key' = 'value', ...) - for (k, v) in table_options { - builder = builder.option(k, v); - } - - let schema = builder.build().map_err(to_datafusion_error)?; + let schema = create_table_schema( + ct, + partition_keys, + enable_ident_normalization, + None, + extract_options(&ct.table_options)?, + )?; catalog .create_table(&identifier, schema, ct.if_not_exists) @@ -1752,7 +1645,7 @@ impl SQLContext { ) }); if has_partition_operation { - Self::ensure_partition_command_target(name, "ALTER TABLE")?; + Self::ensure_main_branch_base_table(name, "ALTER TABLE")?; } else { Self::ensure_main_branch_write_target(name, "ALTER TABLE")?; } @@ -2874,7 +2767,7 @@ impl SQLContext { Ok(()) } - pub(crate) fn ensure_partition_command_target( + pub(crate) fn ensure_main_branch_base_table( name: &ObjectName, operation: &str, ) -> DFResult<()> { @@ -2909,6 +2802,86 @@ impl SQLContext { } } +fn create_table_schema( + ct: &CreateTable, + partition_keys: Vec, + enable_ident_normalization: bool, + inferred_schema: Option<&Schema>, + table_options: HashMap, +) -> DFResult { + let mut builder = paimon::spec::Schema::builder(); + let mut comments = Vec::new(); + if let Some(comment) = &ct.comment { + comments.push(comment); + } + let options = match &ct.table_options { + CreateTableOptions::With(options) + | CreateTableOptions::Options(options) + | CreateTableOptions::TableProperties(options) + | CreateTableOptions::Plain(options) => options.as_slice(), + CreateTableOptions::None => &[], + }; + for option in options { + if let SqlOption::Comment(comment) = option { + comments.push(comment); + } + } + if comments.len() > 1 { + return Err(DataFusionError::Plan("Duplicate table COMMENT".into())); + } + if let Some(CommentDef::WithEq(comment) | CommentDef::WithoutEq(comment)) = + comments.first().copied() + { + builder = builder.comment(Some(comment.clone())); + } + let column_names: Vec = if let Some(schema) = inferred_schema { + for field in schema.fields() { + let data_type = + paimon::arrow::arrow_field_to_paimon_type(field).map_err(to_datafusion_error)?; + // A projection can repeat source field IDs. Assign fresh IDs, including + // nested fields, to the new table through SchemaBuilder. + builder = builder.column(field.name(), data_type); + } + schema + .fields() + .iter() + .map(|field| field.name().clone()) + .collect() + } else { + let names: Vec = ct + .columns + .iter() + .map(|column| normalize_schema_identifier(&column.name, enable_ident_normalization)) + .collect(); + for (col, name) in ct.columns.iter().zip(&names) { + builder = builder.column_with_description( + name.clone(), + column_def_to_paimon_type(col)?, + column_def_comment(col), + ); + } + names + }; + + builder = builder.primary_key(extract_primary_keys(ct, enable_ident_normalization)?); + + if !partition_keys.is_empty() { + for pk in &partition_keys { + if !column_names.contains(pk) { + return Err(DataFusionError::Plan(format!( + "PARTITIONED BY column '{pk}' is not defined in the table" + ))); + } + } + builder = builder.partition_keys(partition_keys); + } + + builder + .options(table_options) + .build() + .map_err(to_datafusion_error) +} + fn typed_view_query(view: &paimon::catalog::View) -> DFResult { let fields = view.schema().fields(); if fields.is_empty() { @@ -3157,6 +3130,7 @@ fn parse_sql_statements(sql: &str) -> DFResult<(Vec, bool)> { let significant = tokens .iter() .enumerate() + .skip_while(|(_, token)| matches!(token.token, Token::Whitespace(_) | Token::SemiColon)) .filter_map(|(index, token)| match &token.token { Token::Whitespace(_) => None, Token::Word(word) => Some((index, word.keyword)), @@ -3199,6 +3173,20 @@ fn parse_sql_statements(sql: &str) -> DFResult<(Vec, bool)> { let (mut statements, vector_changed) = crate::vector_type::parse_statements(tokens) .map_err(|error| DataFusionError::Plan(format!("SQL parse error: {error}")))?; crate::geospatial::quote_cast_types(&mut statements); + // sqlparser consumes some modifiers, such as OR ALTER, without retaining + // them in CreateTable. Validate the original prefix as well as the AST. + if matches!(statements.first(), Some(Statement::CreateTable(ct)) if !ct.temporary && ct.query.is_some()) + { + if !matches!( + significant.as_slice(), + [(_, Keyword::CREATE), (_, Keyword::TABLE), ..] + ) { + return Err(DataFusionError::NotImplemented( + "CREATE TABLE AS SELECT does not support modifiers before TABLE".into(), + )); + } + validate_ctas_option_clauses(sql)?; + } if create_function_if_not_exists { let Some(Statement::CreateFunction(create_function)) = statements.first_mut() else { return Err(DataFusionError::Plan( @@ -3322,284 +3310,145 @@ fn validate_persistent_create_view(create_view: &CreateView) -> DFResult<()> { Ok(()) } -/// Quick check whether the SQL looks like a CREATE TABLE statement. -/// Skips leading whitespace, `--` line comments, and `/* */` block comments. -fn looks_like_create_table(sql: &str) -> bool { - let bytes = sql.as_bytes(); - let len = bytes.len(); - let mut i = 0; - // Skip leading whitespace and comments - loop { - while i < len && bytes[i].is_ascii_whitespace() { - i += 1; - } - if i + 1 < len && bytes[i] == b'-' && bytes[i + 1] == b'-' { - i += 2; - while i < len && bytes[i] != b'\n' { - i += 1; - } - continue; - } - if i + 1 < len && bytes[i] == b'/' && bytes[i + 1] == b'*' { - i += 2; - while i + 1 < len { - if bytes[i] == b'*' && bytes[i + 1] == b'/' { - i += 2; - break; - } - i += 1; - } - continue; - } - break; - } - // Match "CREATE" then whitespace then optional "TEMPORARY"/"TEMP" then "TABLE" (all ASCII, byte-safe) - if i + 6 > len || !bytes[i..i + 6].eq_ignore_ascii_case(b"CREATE") { - return false; - } - i += 6; - if i >= len || !bytes[i].is_ascii_whitespace() { - return false; - } - while i < len && bytes[i].is_ascii_whitespace() { - i += 1; - } - // Skip optional TEMPORARY or TEMP keyword - if i + 9 <= len && bytes[i..i + 9].eq_ignore_ascii_case(b"TEMPORARY") { - i += 9; - while i < len && bytes[i].is_ascii_whitespace() { - i += 1; - } - } else if i + 4 <= len && bytes[i..i + 4].eq_ignore_ascii_case(b"TEMP") { - i += 4; - while i < len && bytes[i].is_ascii_whitespace() { - i += 1; - } - } - // After optional TEMPORARY/TEMP, reject CREATE TEMPORARY VIEW / CREATE TEMP VIEW - if i + 4 <= len && bytes[i..i + 4].eq_ignore_ascii_case(b"VIEW") { - return false; - } - i + 5 <= len && bytes[i..i + 5].eq_ignore_ascii_case(b"TABLE") -} - -/// Find `PARTITIONED BY` keyword position, skipping string literals and comments. -fn find_partitioned_by(sql: &str) -> Option<(usize, usize)> { - let bytes = sql.as_bytes(); - let len = bytes.len(); - let mut i = 0; - while i < len { - match bytes[i] { - b'\'' => { - i += 1; - while i < len { - if bytes[i] == b'\'' { - i += 1; - if i < len && bytes[i] == b'\'' { - i += 1; - } else { - break; - } - } else { - i += 1; - } - } - } - b'-' if i + 1 < len && bytes[i + 1] == b'-' => { - i += 2; - while i < len && bytes[i] != b'\n' { - i += 1; - } - } - b'/' if i + 1 < len && bytes[i + 1] == b'*' => { - i += 2; - while i + 1 < len { - if bytes[i] == b'*' && bytes[i + 1] == b'/' { - i += 2; - break; - } - i += 1; - } - } - b if b.is_ascii_alphabetic() && i + 11 <= len => { - if bytes[i..i + 11].eq_ignore_ascii_case(b"PARTITIONED") { - let rest = &bytes[i + 11..]; - let ws = rest.iter().take_while(|b| b.is_ascii_whitespace()).count(); - if ws > 0 - && i + 11 + ws + 2 <= len - && rest[ws..ws + 2].eq_ignore_ascii_case(b"BY") - { - let by_end = i + 11 + ws + 2; - return Some((i, by_end)); - } - } - i += 1; - } - _ => { - i += 1; - } +fn create_table_parser(sql: &str) -> Option> { + if !matches!( + sql.trim_start().as_bytes().first(), + Some(b'c' | b'C' | b'-' | b'/' | b';') + ) { + return None; + } + let mut parser = Parser::new(&GenericDialect).try_with_sql(sql).ok()?; + while parser.consume_token(&Token::SemiColon) {} + if !parser.parse_keyword(Keyword::CREATE) { + return None; + } + while let Token::Word(word) = parser.next_token().token { + match word.keyword { + Keyword::TABLE => return Some(parser), + Keyword::OR + | Keyword::REPLACE + | Keyword::ALTER + | Keyword::TEMP + | Keyword::TEMPORARY + | Keyword::EXTERNAL + | Keyword::GLOBAL + | Keyword::LOCAL + | Keyword::TRANSIENT + | Keyword::VOLATILE + | Keyword::DYNAMIC + | Keyword::ICEBERG + | Keyword::SNAPSHOT => {} + _ => break, } } None } -/// Parse a single partition column token, handling quoted identifiers. -fn parse_partition_column(token: &str, enable_ident_normalization: bool) -> DFResult { - let trimmed = token.trim(); - if trimmed.is_empty() { - return Err(DataFusionError::Plan( - "Empty column name in PARTITIONED BY".to_string(), - )); - } - - let first = trimmed.as_bytes()[0]; - if first == b'"' || first == b'`' { - let mut value = String::new(); - let mut end = None; - let mut chars = trimmed[1..].char_indices().peekable(); - while let Some((idx, ch)) = chars.next() { - if ch == first as char { - if chars.peek().is_some_and(|(_, next)| *next == first as char) { - value.push(ch); - chars.next(); - } else { - end = Some(1 + idx + ch.len_utf8()); - break; +fn validate_ctas_option_clauses(sql: &str) -> DFResult<()> { + // sqlparser retains only the last options clause, including empty clauses. + let Some(mut parser) = create_table_parser(sql) else { + return Ok(()); + }; + let _ = parser.parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]); + parser.parse_object_name(false)?; + let mut depth = 0; + let mut seen_options = false; + loop { + match parser.next_token().token { + Token::EOF => return Ok(()), + Token::LParen => depth += 1, + Token::RParen => depth -= 1, + Token::Word(word) if depth == 0 => match word.keyword { + Keyword::AS => return Ok(()), + Keyword::WITH | Keyword::TBLPROPERTIES | Keyword::OPTIONS + if parser.peek_token_ref().token == Token::LParen => + { + if seen_options { + return Err(plan_datafusion_err!( + "CREATE TABLE AS SELECT accepts only one table options clause" + )); + } + seen_options = true; } - } else { - value.push(ch); - } - } - if let Some(end) = end { - if trimmed[end..].trim().is_empty() { - return Ok(normalize_schema_identifier( - &Ident::with_quote(first as char, value), - enable_ident_normalization, - )); - } + _ => {} + }, + _ => {} } - return Err(DataFusionError::Plan(format!( - "Invalid quoted identifier in PARTITIONED BY: {trimmed}" - ))); - } - - let parts: Vec<&str> = trimmed.split_whitespace().collect(); - match parts.len() { - 1 => Ok(normalize_schema_identifier( - &Ident::new(parts[0]), - enable_ident_normalization, - )), - _ => Err(DataFusionError::Plan(format!( - "PARTITIONED BY column '{}' should not specify a type. \ - Use column references only, e.g. PARTITIONED BY ({})", - parts[0], parts[0] - ))), } } -fn split_partition_columns(inner: &str) -> DFResult> { - let mut columns = Vec::new(); - let mut start = 0; - let mut quote = None; - let mut chars = inner.char_indices().peekable(); - while let Some((idx, ch)) = chars.next() { - match quote { - Some(q) if ch == q => { - if chars.peek().is_some_and(|(_, next)| *next == q) { - chars.next(); - } else { - quote = None; - } - } - Some(_) => {} - None if ch == '"' || ch == '`' => quote = Some(ch), - None if ch == ',' => { - columns.push(&inner[start..idx]); - start = idx + ch.len_utf8(); - } - None => {} +fn sql_byte_offset(sql: &str, location: Location) -> usize { + let mut offset = 0; + for (index, line) in sql.split_inclusive('\n').enumerate() { + if index + 1 == location.line as usize { + return offset + + line + .char_indices() + .nth(location.column as usize - 1) + .map_or(line.len(), |(index, _)| index); } + offset += line.len(); } - if quote.is_some() { - return Err(DataFusionError::Plan( - "Unterminated quoted identifier in PARTITIONED BY".to_string(), - )); - } - columns.push(&inner[start..]); - Ok(columns) + offset } -/// Extract `PARTITIONED BY (col1, col2, ...)` from SQL before parsing. -/// -/// Paimon only allows column references (no types) in PARTITIONED BY. -/// Since sqlparser's GenericDialect requires types in column definitions, -/// we extract and validate the clause ourselves, then strip it from the SQL -/// so sqlparser can parse the rest. +/// GenericDialect expects typed Hive partition columns. Parse Paimon's list of +/// column references separately and leave the AS query intact. fn extract_partition_by( sql: &str, enable_ident_normalization: bool, ) -> DFResult<(String, Vec)> { - let Some((kw_start, by_end)) = find_partitioned_by(sql) else { - return Ok((sql.to_string(), vec![])); + let Some(mut parser) = create_table_parser(sql) else { + return Ok((sql.to_owned(), vec![])); }; - - let after_by = sql[by_end..].trim_start(); - let paren_start = by_end + (sql[by_end..].len() - after_by.len()); - - if !after_by.starts_with('(') { - return Err(DataFusionError::Plan( - "Expected '(' after PARTITIONED BY".to_string(), - )); - } - - let inner_start = paren_start + 1; - let mut depth = 1; - let mut paren_end = None; - let mut quote = None; - let mut chars = sql[inner_start..].char_indices().peekable(); - while let Some((i, ch)) = chars.next() { - match quote { - Some(q) if ch == q => { - if chars.peek().is_some_and(|(_, next)| *next == q) { - chars.next(); - } else { - quote = None; - } + let mut depth = 0; + let start = loop { + let token = parser.next_token(); + match token.token { + Token::LParen => depth += 1, + Token::RParen => depth -= 1, + Token::Word(word) if depth == 0 && word.keyword == Keyword::AS => { + return Ok((sql.to_owned(), vec![])); } - Some(_) => {} - None if ch == '"' || ch == '`' => quote = Some(ch), - None if ch == '(' => depth += 1, - None if ch == ')' => { - depth -= 1; - if depth == 0 { - paren_end = Some(inner_start + i); - break; - } + Token::Word(word) + if depth == 0 + && word.keyword == Keyword::PARTITIONED + && parser.parse_keyword(Keyword::BY) => + { + break token.span.start; } - None => {} + Token::EOF => return Ok((sql.to_owned(), vec![])), + _ => {} } + }; + if !parser.consume_token(&Token::LParen) { + return Err(plan_datafusion_err!("Expected '(' after PARTITIONED BY")); } - let paren_end = paren_end.ok_or_else(|| { - DataFusionError::Plan("Unmatched '(' in PARTITIONED BY clause".to_string()) - })?; - - let inner = sql[inner_start..paren_end].trim(); - if inner.is_empty() { - return Err(DataFusionError::Plan( - "PARTITIONED BY must specify at least one column".to_string(), + if parser.peek_token_ref().token == Token::RParen { + return Err(plan_datafusion_err!( + "PARTITIONED BY must specify at least one column" )); } - - let mut partition_keys = Vec::new(); - for token in split_partition_columns(inner)? { - partition_keys.push(parse_partition_column(token, enable_ident_normalization)?); + let keys = parser.parse_comma_separated(Parser::parse_identifier)?; + let end = parser.next_token(); + match end.token { + Token::RParen => {} + Token::EOF => return Err(plan_datafusion_err!("Unmatched '(' in PARTITIONED BY clause")), + other => return Err(plan_datafusion_err!( + "Expected ',' or ')' in PARTITIONED BY, found {other}. Partition columns should not specify a type" + )), } - - let clause_end = paren_end + 1; - let mut rewritten = String::with_capacity(sql.len()); - rewritten.push_str(&sql[..kw_start]); - rewritten.push_str(&sql[clause_end..]); - Ok((rewritten, partition_keys)) + let normalizer = IdentNormalizer::new(enable_ident_normalization); + let keys = keys + .into_iter() + .map(|key| normalizer.normalize(key)) + .collect(); + let mut rewritten = sql.to_owned(); + rewritten.replace_range( + sql_byte_offset(sql, start)..sql_byte_offset(sql, end.span.end), + "", + ); + Ok((rewritten, keys)) } pub(crate) fn normalize_schema_identifier( @@ -3985,27 +3834,27 @@ fn object_name_to_single_identifier( } /// Extract key-value pairs from [`CreateTableOptions`]. -fn extract_options(opts: &CreateTableOptions) -> DFResult> { +fn extract_options(opts: &CreateTableOptions) -> DFResult> { let sql_options = match opts { CreateTableOptions::With(options) | CreateTableOptions::Options(options) | CreateTableOptions::TableProperties(options) | CreateTableOptions::Plain(options) => options, - CreateTableOptions::None => return Ok(Vec::new()), + CreateTableOptions::None => return Ok(HashMap::new()), }; sql_options .iter() .filter(|option| !matches!(option, SqlOption::Comment(_))) .map(|opt| match opt { SqlOption::KeyValue { key, value } => { - let v = value.to_string(); - // Strip surrounding quotes from the value if present. - let v = v - .strip_prefix('\'') - .and_then(|s| s.strip_suffix('\'')) - .unwrap_or(&v) - .to_string(); - Ok((key.value.clone(), v)) + let value = match value { + SqlExpr::Value(literal) => literal + .clone() + .into_string() + .unwrap_or_else(|| literal.to_string()), + expr => expr.to_string(), + }; + Ok((key.value.clone(), value)) } other => Err(DataFusionError::Plan(format!( "Unsupported table option: {other}" @@ -4429,284 +4278,112 @@ fn datum_to_constant_array( } } -struct VersionAsOfInfo { - table_name: String, - version: String, - /// Byte range (start, end) covering "table_name VERSION AS OF n" - clause_range: (usize, usize), -} - -struct TimestampAsOfInfo { - table_name: String, - timestamp: String, - /// Byte range (start, end) covering "table_name TIMESTAMP AS OF 'ts'" - clause_range: (usize, usize), +struct TimeTravelSource { + binding: String, + table: TableReference, + clause: TimeTravelClause, } /// A parsed `VERSION AS OF` or `TIMESTAMP AS OF` selector. +#[derive(Debug, PartialEq)] enum TimeTravelClause { Version(String), - TimestampMillis(i64), + Timestamp(String), } -/// Locate the table before a time-travel keyword without treating delimiters -/// inside quoted identifiers as boundaries. -fn time_travel_table_before(prefix: &str) -> (usize, &str) { - let prefix = prefix.trim_end(); - let mut quote = None; - for (index, ch) in prefix.char_indices().rev() { - if let Some(delimiter) = quote { - if ch == delimiter { - quote = None; - } - } else if matches!(ch, '"' | '`') { - // Adjacent escaped quotes toggle the quote state twice. - quote = Some(ch); - } else if ch.is_whitespace() || matches!(ch, ',' | '(') { - let start = index + ch.len_utf8(); - return (start, &prefix[start..]); - } +/// Defer snapshot lookups so `IF NOT EXISTS` can skip the source query. +fn mask_time_travel_sources(sql: &mut String, normalize: bool) -> Vec { + let lowercase = sql.to_ascii_lowercase(); + if !lowercase.contains("version") && !lowercase.contains("timestamp") { + return Vec::new(); } - (0, prefix) -} - -/// Check for a time-travel keyword outside quoted strings, identifiers, and comments. -fn contains_time_travel_keyword(sql: &str) -> bool { - let bytes = sql.as_bytes(); - let len = bytes.len(); - let mut i = 0; - while i < len { - match bytes[i] { - quote @ (b'\'' | b'"' | b'`') => { - // Skip a quoted string or identifier, including doubled quotes. - i += 1; - while i < len { - if bytes[i] == quote { - i += 1; - if i < len && bytes[i] == quote { - i += 1; // escaped quote - } else { - break; - } - } else { - i += 1; - } - } - } - b'-' if i + 1 < len && bytes[i + 1] == b'-' => { - // Skip line comment - i += 2; - while i < len && bytes[i] != b'\n' { - i += 1; - } + let tokens: Vec<_> = Tokenizer::new(&GenericDialect, sql) + .tokenize_with_location() + .unwrap_or_default() + .into_iter() + .filter(|token| !matches!(token.token, Token::Whitespace(_))) + .collect(); + let mut sources = Vec::new(); + for (index, clause) in tokens.windows(4).enumerate().rev() { + if index == 0 + || !matches!(tokens[index - 1].token, Token::Word(_)) + || !matches!(&clause[1].token, Token::Word(word) if word.keyword == Keyword::AS) + || !matches!(&clause[2].token, Token::Word(word) if word.keyword == Keyword::OF) + { + continue; + } + let Token::Word(word) = &clause[0].token else { + continue; + }; + let selector = match (word.keyword, &clause[3].token) { + (Keyword::VERSION, Token::SingleQuotedString(tag)) => { + TimeTravelClause::Version(tag.clone()) } - b'/' if i + 1 < len && bytes[i + 1] == b'*' => { - // Skip block comment - i += 2; - while i + 1 < len { - if bytes[i] == b'*' && bytes[i + 1] == b'/' { - i += 2; - break; - } - i += 1; - } + (Keyword::VERSION, Token::Number(id, false)) + if id.bytes().all(|byte| byte.is_ascii_digit()) => + { + TimeTravelClause::Version(id.clone()) } - _ => { - // Check for keywords - if i + 14 <= len && bytes[i..i + 14].eq_ignore_ascii_case(b"version as of ") { - return true; - } - if i + 16 <= len && bytes[i..i + 16].eq_ignore_ascii_case(b"timestamp as of ") { - return true; - } - i += 1; + (Keyword::TIMESTAMP, Token::SingleQuotedString(timestamp)) => { + TimeTravelClause::Timestamp(timestamp.clone()) } - } + _ => continue, + }; + let mut start = index - 1; + while start >= 2 + && matches!(tokens[start - 1].token, Token::Period) + && matches!(tokens[start - 2].token, Token::Word(_)) + { + start -= 2; + } + let table_start = sql_byte_offset(sql, tokens[start].span.start); + let table_end = sql_byte_offset(sql, tokens[index - 1].span.end); + let table = TableReference::parse_str_normalized(&sql[table_start..table_end], !normalize); + let end = sql_byte_offset(sql, clause[3].span.end); + let binding = format!("__paimon_tt_{}", uuid::Uuid::new_v4().as_simple()); + sql.replace_range(table_start..end, &binding); + sources.push(TimeTravelSource { + binding, + table, + clause: selector, + }); } - false + sources.reverse(); + sources } -/// Extract **all** `VERSION AS OF ` or `VERSION AS OF ''` clauses from a -/// SQL string, skipping quoted strings, identifiers, and comments. -fn extract_all_version_as_of(sql: &str) -> Vec { - let bytes = sql.as_bytes(); - let len = bytes.len(); - let mut i = 0; - let mut results = Vec::new(); - - while i < len { - match bytes[i] { - quote @ (b'\'' | b'"' | b'`') => { - // Skip a quoted string or identifier, including doubled quotes. - i += 1; - while i < len { - if bytes[i] == quote { - i += 1; - if i < len && bytes[i] == quote { - i += 1; // escaped quote - } else { - break; - } - } else { - i += 1; - } - } - } - b'-' if i + 1 < len && bytes[i + 1] == b'-' => { - // Skip line comment - i += 2; - while i < len && bytes[i] != b'\n' { - i += 1; - } - } - b'/' if i + 1 < len && bytes[i + 1] == b'*' => { - // Skip block comment - i += 2; - while i + 1 < len { - if bytes[i] == b'*' && bytes[i + 1] == b'/' { - i += 2; - break; - } - i += 1; - } - } - _ => { - if i + 14 <= len && bytes[i..i + 14].eq_ignore_ascii_case(b"version as of ") { - let kw_start = i; - let val_start = i + 14; - let remaining = &sql[val_start..]; - - // Parse either a quoted tag name or a numeric snapshot ID - let version = if let Some(after_quote) = remaining.strip_prefix('\'') { - // Tag name: VERSION AS OF 'tagname' - if let Some(close_quote) = after_quote.find('\'') { - after_quote[..close_quote].to_string() - } else { - i += 1; - continue; - } - } else { - // Numeric snapshot ID: VERSION AS OF 1 - let v: String = remaining - .chars() - .take_while(|c| c.is_ascii_digit()) - .collect(); - if v.is_empty() { - i += 1; - continue; - } - v - }; - - let is_quoted = remaining.starts_with('\''); - let val_end = if is_quoted { - val_start + version.len() + 2 // 2 quotes - } else { - val_start + version.len() - }; - - let (table_start, table_name) = time_travel_table_before(&sql[..kw_start]); - - if !table_name.is_empty() { - results.push(VersionAsOfInfo { - table_name: table_name.to_string(), - version, - clause_range: (table_start, val_end), - }); - } - - i = val_end; - } else { - i += 1; - } +fn validate_time_travel_sources( + statement: &Statement, + sources: &[TimeTravelSource], +) -> DFResult<()> { + use datafusion::sql::sqlparser::ast::{Visit, Visitor}; + use std::ops::ControlFlow; + + struct Bindings<'a>(HashSet<&'a str>); + impl Visitor for Bindings<'_> { + type Break = (); + fn pre_visit_table_factor(&mut self, factor: &TableFactor) -> ControlFlow<()> { + if let TableFactor::Table { name, .. } = factor { + self.0.remove(name.to_string().as_str()); } + ControlFlow::Continue(()) } } - - results -} - -/// Extract all time-travel timestamp clauses outside quoted strings, -/// identifiers, and comments. -fn extract_all_timestamp_as_of(sql: &str) -> Vec { - let bytes = sql.as_bytes(); - let len = bytes.len(); - let mut i = 0; - let mut results = Vec::new(); - - while i < len { - match bytes[i] { - quote @ (b'\'' | b'"' | b'`') => { - // Skip a quoted string or identifier, including doubled quotes. - i += 1; - while i < len { - if bytes[i] == quote { - i += 1; - if i < len && bytes[i] == quote { - i += 1; // escaped quote - } else { - break; - } - } else { - i += 1; - } - } - } - b'-' if i + 1 < len && bytes[i + 1] == b'-' => { - // Skip line comment - i += 2; - while i < len && bytes[i] != b'\n' { - i += 1; - } - } - b'/' if i + 1 < len && bytes[i + 1] == b'*' => { - // Skip block comment - i += 2; - while i + 1 < len { - if bytes[i] == b'*' && bytes[i + 1] == b'/' { - i += 2; - break; - } - i += 1; - } - } - _ => { - if i + 16 <= len && bytes[i..i + 16].eq_ignore_ascii_case(b"timestamp as of ") { - let kw_start = i; - let val_start = i + 16; - let remaining = &sql[val_start..]; - - // Read the quoted timestamp string - if !remaining.starts_with('\'') { - i += 1; - continue; - } - if let Some(close_quote) = remaining[1..].find('\'') { - let timestamp = remaining[1..close_quote + 1].to_string(); - let val_end = val_start + close_quote + 2; // skip both quotes - - let (table_start, table_name) = time_travel_table_before(&sql[..kw_start]); - - if !table_name.is_empty() { - results.push(TimestampAsOfInfo { - table_name: table_name.to_string(), - timestamp, - clause_range: (table_start, val_end), - }); - } - - i = val_end; - } else { - i += 1; - } - } else { - i += 1; - } - } + let mut bindings = Bindings(sources.iter().map(|s| s.binding.as_str()).collect()); + match statement { + Statement::CreateTable(ct) => { + let _ = ct.query.visit(&mut bindings); + } + _ => { + let _ = statement.visit(&mut bindings); } } - - results + if !bindings.0.is_empty() { + return Err(DataFusionError::Plan( + "Time travel requires a source table reference".into(), + )); + } + Ok(()) } /// Return an empty DataFrame with a single "result" column containing "OK". @@ -8723,15 +8400,13 @@ mod tests { #[test] fn test_extract_options_with_kv() { - // Parse a CREATE TABLE with WITH options to get a real CreateTableOptions let dialect = GenericDialect {}; let stmts = Parser::parse_sql(&dialect, "CREATE TABLE t (id INT) WITH ('bucket' = '4')").unwrap(); if let Statement::CreateTable(ct) = &stmts[0] { let opts = extract_options(&ct.table_options).unwrap(); assert_eq!(opts.len(), 1); - assert_eq!(opts[0].0, "bucket"); - assert_eq!(opts[0].1, "4"); + assert_eq!(opts["bucket"], "4"); } else { panic!("expected CreateTable"); } @@ -9587,12 +9262,18 @@ mod tests { #[test] fn test_extract_partition_by_skips_string_literal() { - let sql = - "CREATE TABLE t (id INT) WITH ('note' = 'PARTITIONED BY (x)') PARTITIONED BY (id)"; - let (rewritten, keys) = extract_partition_by(sql, true).unwrap(); - assert_eq!(keys, vec!["id"]); - assert!(rewritten.contains("WITH")); - assert!(rewritten.contains("'PARTITIONED BY (x)'")); + for literal in [ + "'PARTITIONED BY (x)'", + "$$PARTITIONED BY (x)$$", + "$tag$PARTITIONED BY (x)$tag$", + ] { + let sql = + format!("CREATE TABLE t (id INT) WITH ('note' = {literal}) PARTITIONED BY (id)"); + let (rewritten, keys) = extract_partition_by(&sql, true).unwrap(); + assert_eq!(keys, vec!["id"]); + assert!(rewritten.contains("WITH")); + assert!(rewritten.contains(literal)); + } } #[test] @@ -9613,10 +9294,10 @@ mod tests { } #[test] - fn test_extract_partition_by_double_quoted_identifier_with_escaped_quote_and_comma() { + fn test_extract_partition_by_quoted_identifiers_and_comments() { let (_, keys) = extract_partition_by( "CREATE TABLE t (\"a\"\"b,c\" INT, `d``e,f` INT) \ - PARTITIONED BY (\"a\"\"b,c\", `d``e,f`)", + PARTITIONED BY /* layout */ (\"a\"\"b,c\" /* ,) */, -- second key\n `d``e,f`)", true, ) .unwrap(); @@ -9650,33 +9331,55 @@ mod tests { #[test] fn test_extract_partition_by_skips_block_comment() { - let sql = "CREATE TABLE t (id INT) /* PARTITIONED BY (x) */ PARTITIONED BY (id)"; - let (rewritten, keys) = extract_partition_by(sql, true).unwrap(); - assert_eq!(keys, vec!["id"]); - assert!(rewritten.contains("/* PARTITIONED BY (x) */")); + for comment in [ + "/* PARTITIONED BY (x) */", + "/* outer /* inner */ PARTITIONED BY (x) */", + ] { + let sql = + format!("CREATE TABLE \"中文\" (id INT) {comment}\nPARTITIONED /* key */ BY (id)"); + let (rewritten, keys) = extract_partition_by(&sql, true).unwrap(); + assert_eq!(keys, vec!["id"]); + assert!(rewritten.contains(comment)); + assert!(rewritten.starts_with("CREATE TABLE \"中文\" (id INT)")); + } } #[test] - fn test_looks_like_create_table() { - assert!(looks_like_create_table("CREATE TABLE t (id INT)")); - assert!(looks_like_create_table(" create table t (id INT)")); - assert!(looks_like_create_table( + fn test_create_table_parser() { + for sql in [ + "CREATE TABLE t (id INT)", + " create table t (id INT)", "CREATE TABLE IF NOT EXISTS t (id INT)", - )); - assert!(looks_like_create_table( "/* note */ CREATE TABLE t (id INT)", - )); - assert!(looks_like_create_table( "-- comment\nCREATE TABLE t (id INT)", - )); - assert!(looks_like_create_table( "/* a */ /* b */ CREATE TABLE t (id INT)", - )); - assert!(!looks_like_create_table("ALTER TABLE t ADD COLUMN x INT")); - assert!(!looks_like_create_table("SELECT 1")); - assert!(!looks_like_create_table( + "CREATE /* note */ TABLE t AS SELECT 1", + "CREATE TEMP TABLE t AS SELECT 1", + "CREATE TEMPORARY TABLE t AS SELECT 1", + "CREATE OR REPLACE TABLE t AS SELECT * FROM s VERSION AS OF 1", + "CREATE OR ALTER TABLE t AS SELECT * FROM s VERSION AS OF 1", + "CREATE EXTERNAL TABLE t AS SELECT 1", + "CREATE GLOBAL TABLE t AS SELECT 1", + "CREATE TRANSIENT TABLE t AS SELECT 1", + "; CREATE TABLE t AS SELECT 1", + "/* start */ ; ; -- empty statements\n; CREATE TABLE t AS SELECT 1", + "; CREATE OR REPLACE TABLE t AS SELECT * FROM s VERSION AS OF 1", + ] { + assert!(create_table_parser(sql).is_some(), "{sql}"); + } + for sql in [ + "ALTER TABLE t ADD COLUMN x INT", + "SELECT 1", "SELECT aaaaaaaaaaaaaaaaaaaa中文 FROM t", - )); + "CREATE VIEW v AS SELECT 1", + "CREATE TEMP VIEW v AS SELECT 1", + "CREATE TABLE_SUFFIX t (id INT)", + "CREATE \"TABLE\" t (id INT)", + "; ; -- only empty statements\n;", + "; SELECT 1; CREATE TABLE t AS SELECT 1", + ] { + assert!(create_table_parser(sql).is_none(), "{sql}"); + } } // ==================== partition key validation tests ==================== @@ -10700,212 +10403,81 @@ mod tests { } #[test] - fn test_extract_version_as_of() { - let sql = "SELECT id, name FROM paimon.default.time_travel_table VERSION AS OF 1"; - let infos = extract_all_version_as_of(sql); - assert_eq!(infos.len(), 1); - let info = &infos[0]; - assert_eq!(info.version, "1"); - assert_eq!(info.table_name, "paimon.default.time_travel_table"); - let rewritten = format!( - "{}__uuid{}", - &sql[..info.clause_range.0], - &sql[info.clause_range.1..] - ); - assert_eq!(rewritten, "SELECT id, name FROM __uuid"); - } - - #[test] - fn test_extract_version_as_of_multi_digit() { - let sql = "SELECT * FROM mydb.t VERSION AS OF 42"; - let infos = extract_all_version_as_of(sql); - assert_eq!(infos.len(), 1); - let info = &infos[0]; - assert_eq!(info.version, "42"); - assert_eq!(info.table_name, "mydb.t"); - let rewritten = format!( - "{}__uuid{}", - &sql[..info.clause_range.0], - &sql[info.clause_range.1..] - ); - assert_eq!(rewritten, "SELECT * FROM __uuid"); - } - - #[test] - fn test_extract_version_as_of_case_insensitive() { - let sql = "SELECT * FROM t version as of 5"; - let infos = extract_all_version_as_of(sql); - assert_eq!(infos.len(), 1); - let info = &infos[0]; - assert_eq!(info.version, "5"); - assert_eq!(info.table_name, "t"); - let rewritten = format!( - "{}__uuid{}", - &sql[..info.clause_range.0], - &sql[info.clause_range.1..] - ); - assert_eq!(rewritten, "SELECT * FROM __uuid"); - } - - #[test] - fn test_extract_version_as_of_not_present() { - let sql = "SELECT * FROM t"; - assert!(extract_all_version_as_of(sql).is_empty()); - } - - #[test] - fn test_extract_version_as_of_tag() { - let sql = "SELECT id, name FROM paimon.default.t VERSION AS OF 'snapshot1'"; - let infos = extract_all_version_as_of(sql); - assert_eq!(infos.len(), 1); - let info = &infos[0]; - assert_eq!(info.version, "snapshot1"); - assert_eq!(info.table_name, "paimon.default.t"); - let rewritten = format!( - "{}__uuid{}", - &sql[..info.clause_range.0], - &sql[info.clause_range.1..] - ); - assert_eq!(rewritten, "SELECT id, name FROM __uuid"); - } - - #[test] - fn test_extract_version_as_of_tag_case_insensitive() { - let sql = "SELECT * FROM t version as of 'my_tag'"; - let infos = extract_all_version_as_of(sql); - assert_eq!(infos.len(), 1); - let info = &infos[0]; - assert_eq!(info.version, "my_tag"); - assert_eq!(info.table_name, "t"); - let rewritten = format!( - "{}__uuid{}", - &sql[..info.clause_range.0], - &sql[info.clause_range.1..] - ); - assert_eq!(rewritten, "SELECT * FROM __uuid"); - } - - #[test] - fn test_extract_version_as_of_numeric_still_works() { - let sql = "SELECT * FROM t VERSION AS OF 123"; - let infos = extract_all_version_as_of(sql); - assert_eq!(infos.len(), 1); - assert_eq!(infos[0].version, "123"); - assert_eq!(infos[0].table_name, "t"); - } - - #[test] - fn test_extract_version_as_of_multiple() { - // JOIN two time-travel tables - let sql = "SELECT * FROM t1 VERSION AS OF 1 JOIN t2 VERSION AS OF 2 ON t1.id = t2.id"; - let infos = extract_all_version_as_of(sql); - assert_eq!(infos.len(), 2); - assert_eq!(infos[0].version, "1"); - assert_eq!(infos[0].table_name, "t1"); - assert_eq!(infos[1].version, "2"); - assert_eq!(infos[1].table_name, "t2"); - } - - #[test] - fn test_extract_version_as_of_skips_string_literal() { - let sql = "SELECT * FROM t WHERE note = 'version as of 1'"; - let infos = extract_all_version_as_of(sql); - assert!(infos.is_empty()); - } - - #[test] - fn test_extract_version_as_of_skips_comment() { - let sql = "SELECT * FROM t -- version as of 1\n WHERE id > 0"; - let infos = extract_all_version_as_of(sql); - assert!(infos.is_empty()); + fn test_mask_time_travel_sources() { + use TimeTravelClause::{Timestamp, Version}; + + for (selector, expected) in [ + ("VERSION AS OF 1", Version("1".into())), + ("version as of 123", Version("123".into())), + ("VERSION AS OF 'my_tag'", Version("my_tag".into())), + ("VERSION AS OF 'tag''quoted'", Version("tag'quoted".into())), + ( + "/* table */ VERSION\nAS /* selector */ OF 42", + Version("42".into()), + ), + ( + "timestamp as of '2024-01-15 10:30:00'", + Timestamp("2024-01-15 10:30:00".into()), + ), + ( + "/* table */ TIMESTAMP\nAS /* selector */ OF '2024-01-15'", + Timestamp("2024-01-15".into()), + ), + ] { + let mut sql = format!( + r#"SELECT 'İ' FROM paimon."my db"."with ""quote"", (space" {selector} AS s"# + ); + let sources = mask_time_travel_sources(&mut sql, true); + assert_eq!(sources.len(), 1, "{selector}"); + assert_eq!(sources[0].clause, expected); + assert_eq!( + sources[0].table, + TableReference::full("paimon", "my db", "with \"quote\", (space") + ); + assert_eq!(sql, format!("SELECT 'İ' FROM {} AS s", sources[0].binding)); + } } #[test] - fn test_time_travel_keeps_quoted_table_reference() { - let table = r#"paimon."my db"."with ""quote"", (space""#; - let version_sql = format!("SELECT 'İ' FROM {table} VERSION AS OF 1"); - let versions = extract_all_version_as_of(&version_sql); - assert_eq!(versions.len(), 1); - assert_eq!(versions[0].table_name, table); - assert_eq!( - &version_sql[versions[0].clause_range.0..versions[0].clause_range.1], - format!("{table} VERSION AS OF 1") - ); - - let timestamp_sql = format!("SELECT 'İ' FROM {table} TIMESTAMP AS OF '2024-01-01'"); - let timestamps = extract_all_timestamp_as_of(×tamp_sql); - assert_eq!(timestamps.len(), 1); - assert_eq!(timestamps[0].table_name, table); + fn test_mask_multiple_time_travel_sources() { + let mut sql = + "SELECT * FROM t1 VERSION AS OF 1 JOIN t2 VERSION AS OF 2 ON t1.id = t2.id".to_string(); + let sources = mask_time_travel_sources(&mut sql, true); + assert_eq!(sources.len(), 2); + assert_eq!(sources[0].table, TableReference::bare("t1")); + assert_eq!(sources[0].clause, TimeTravelClause::Version("1".into())); + assert_eq!(sources[1].table, TableReference::bare("t2")); + assert_eq!(sources[1].clause, TimeTravelClause::Version("2".into())); assert_eq!( - ×tamp_sql[timestamps[0].clause_range.0..timestamps[0].clause_range.1], - format!("{table} TIMESTAMP AS OF '2024-01-01'") - ); - } - - #[test] - fn test_contains_time_travel_keyword() { - assert!(contains_time_travel_keyword( - "SELECT * FROM t VERSION AS OF 1" - )); - assert!(contains_time_travel_keyword( - "SELECT * FROM t TIMESTAMP AS OF '2024-01-01 00:00:00'" - )); - // Inside string literal — should NOT match - assert!(!contains_time_travel_keyword( - "SELECT * FROM t WHERE note = 'version as of 1'" - )); - // Inside comment — should NOT match - assert!(!contains_time_travel_keyword( - "SELECT * FROM t -- version as of 1" - )); - assert!(!contains_time_travel_keyword( - "SELECT * FROM t /* timestamp as of now */ WHERE id > 0" - )); - assert!(!contains_time_travel_keyword( - r#"SELECT * FROM "version as of 1""# - )); - assert!(!contains_time_travel_keyword( - "SELECT * FROM `timestamp as of now`" - )); - // No keyword at all - assert!(!contains_time_travel_keyword("SELECT * FROM t")); - } - - #[test] - fn test_extract_timestamp_as_of() { - let sql = "SELECT * FROM paimon.default.t TIMESTAMP AS OF '2024-01-15 10:30:00'"; - let infos = extract_all_timestamp_as_of(sql); - assert_eq!(infos.len(), 1); - let info = &infos[0]; - assert_eq!(info.timestamp, "2024-01-15 10:30:00"); - assert_eq!(info.table_name, "paimon.default.t"); - let rewritten = format!( - "{}__uuid{}", - &sql[..info.clause_range.0], - &sql[info.clause_range.1..] - ); - assert_eq!(rewritten, "SELECT * FROM __uuid"); - } - - #[test] - fn test_extract_timestamp_as_of_case_insensitive() { - let sql = "SELECT * FROM t timestamp as of '2024-06-01 00:00:00'"; - let infos = extract_all_timestamp_as_of(sql); - assert_eq!(infos.len(), 1); - let info = &infos[0]; - assert_eq!(info.timestamp, "2024-06-01 00:00:00"); - assert_eq!(info.table_name, "t"); - let rewritten = format!( - "{}__uuid{}", - &sql[..info.clause_range.0], - &sql[info.clause_range.1..] + sql, + format!( + "SELECT * FROM {} JOIN {} ON t1.id = t2.id", + sources[0].binding, sources[1].binding + ) ); - assert_eq!(rewritten, "SELECT * FROM __uuid"); } #[test] - fn test_extract_timestamp_as_of_not_present() { - let sql = "SELECT * FROM t"; - assert!(extract_all_timestamp_as_of(sql).is_empty()); + fn test_time_travel_ignores_literals_comments_and_aliases() { + for sql in [ + "SELECT * FROM t", + "SELECT * FROM t WHERE note = 'version as of 1'", + "SELECT $$source VERSION AS OF 1$$ FROM t", + "SELECT $tag$source TIMESTAMP AS OF '2024-01-01'$tag$ FROM t", + "SELECT * FROM t -- version as of 1", + "SELECT * FROM t /* timestamp as of now */ WHERE id > 0", + "SELECT * FROM t /* outer /* inner */ source VERSION AS OF 1 */", + r#"SELECT * FROM "version as of 1""#, + "SELECT * FROM `timestamp as of now`", + "SELECT conversion AS of_value FROM t", + "SELECT version AS of FROM t", + ] { + let mut masked = sql.to_string(); + assert!( + mask_time_travel_sources(&mut masked, true).is_empty(), + "{sql}" + ); + assert_eq!(masked, sql); + } } } diff --git a/crates/integrations/datafusion/src/sql_context/ctas.rs b/crates/integrations/datafusion/src/sql_context/ctas.rs new file mode 100644 index 000000000..40f827525 --- /dev/null +++ b/crates/integrations/datafusion/src/sql_context/ctas.rs @@ -0,0 +1,282 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Persistent CREATE TABLE AS SELECT. Population failures retain the table +//! because concurrent writers can use it as soon as creation succeeds. + +use std::ops::ControlFlow; +use std::sync::Arc; + +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::error::{DataFusionError, Result as DFResult}; +use datafusion::execution::context::SQLOptions; +use datafusion::execution::TaskContext; +use datafusion::physical_plan::{execute_stream, ExecutionPlan}; +use datafusion::prelude::DataFrame; +use datafusion::sql::planner::object_name_to_table_reference; +use datafusion::sql::sqlparser::ast::helpers::stmt_create_table::CreateTableBuilder; +use datafusion::sql::sqlparser::ast::{ + CreateTable, CreateTableOptions, Query, Select, SqlOption, Statement, Visit, Visitor, +}; +use futures::StreamExt; +use paimon::spec::{CoreOptions, DataType as PaimonDataType, TableType}; +use paimon::table::Table; + +use super::{create_table_schema, extract_options, ok_result, SQLContext, TimeTravelSource}; +use crate::error::to_datafusion_error; +use crate::physical_plan::cast_batch_to_schema; + +impl SQLContext { + pub(super) async fn handle_create_table_as( + &self, + ct: &CreateTable, + query: Box, + partition_keys: Vec, + enable_ident_normalization: bool, + options: SQLOptions, + time_travel_sources: &[TimeTravelSource], + ) -> DFResult { + validate_create_table_as(ct, &query)?; + Self::ensure_main_branch_base_table(&ct.name, "CREATE TABLE AS SELECT")?; + let target = object_name_to_table_reference(ct.name.clone(), enable_ident_normalization)?; + let (catalog, catalog_name, identifier) = self.resolve_table_name_from_ref(&target)?; + + // An external target can exist even when its table engine is unavailable. + match catalog.load_table(&identifier).await { + Ok(_) if ct.if_not_exists => return ok_result(&self.ctx), + Ok(_) => { + return Err(to_datafusion_error(paimon::Error::TableAlreadyExist { + full_name: identifier.full_name(), + })); + } + Err(paimon::Error::TableNotExist { .. }) => {} + Err(error) => return Err(to_datafusion_error(error)), + } + + let table_options = extract_options(&ct.table_options)?; + let table_type = CoreOptions::new(&table_options) + .table_type() + .map_err(to_datafusion_error)?; + if table_type != TableType::Table { + return Err(DataFusionError::NotImplemented(format!( + "CREATE TABLE AS SELECT only supports native Paimon tables, got '{table_type}'" + ))); + } + + let df = Box::pin(self.plan_query(Statement::Query(query), options, time_travel_sources)) + .await?; + let (state, source) = df.into_parts(); + let task_ctx = state.task_ctx(); + // Optimization can narrow nullability beyond the query's declared schema. + let analyzed = + state + .analyzer() + .execute_and_check(source, state.config_options(), |_, _| {})?; + let schema = create_table_schema( + ct, + partition_keys, + enable_ident_normalization, + Some(analyzed.schema().as_arrow()), + table_options, + )?; + if schema.fields().is_empty() { + return Err(DataFusionError::Plan( + "CREATE TABLE AS SELECT requires at least one output column".into(), + )); + } + for field in schema.fields() { + validate_time_precision(field.data_type())?; + } + let write_schema = paimon::arrow::build_target_arrow_schema(schema.fields()) + .map_err(to_datafusion_error)?; + let optimized = state.optimizer().optimize(analyzed, &state, |_, _| {})?; + let plan = state + .query_planner() + .create_physical_plan(&optimized, &state) + .await?; + + // Use strict creation so a concurrent creator's table is never populated. + match catalog.create_table(&identifier, schema, false).await { + Ok(()) => {} + Err(paimon::Error::TableAlreadyExist { .. }) if ct.if_not_exists => { + return ok_result(&self.ctx); + } + Err(error) => return Err(to_datafusion_error(error)), + } + let table = catalog + .get_table(&identifier) + .await + .map_err(to_datafusion_error)?; + + let name = format!("{catalog_name}.{}", identifier.full_name()); + let (mut response, outcome) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + let result = populate_table(table, plan, task_ctx, write_schema, &mut response) + .await + .map_err(|error| { + DataFusionError::Context( + format!( + "CREATE TABLE AS SELECT failed for '{name}'. \ + Inspect the target table before retrying." + ), + Box::new(error), + ) + }); + if let Err(Err(error)) = response.send(result) { + log::warn!("Cancelled CREATE TABLE AS SELECT for '{name}': {error}"); + } + }); + outcome.await.map_err(|error| { + DataFusionError::Execution(format!("CREATE TABLE AS SELECT worker stopped: {error}")) + })??; + ok_result(&self.ctx) + } +} + +async fn populate_table( + table: Table, + plan: Arc, + task_ctx: Arc, + write_schema: SchemaRef, + response: &mut tokio::sync::oneshot::Sender>, +) -> DFResult<()> { + let resources = crate::memory::writer_resources(&task_ctx).map_err(to_datafusion_error)?; + let builder = table.new_write_builder().with_resources(resources); + let commit = builder.try_new_commit().map_err(to_datafusion_error)?; + let mut writer = builder.new_write().map_err(to_datafusion_error)?; + let prepared = async { + let mut input = execute_stream(plan, task_ctx)?; + loop { + let next = tokio::select! { + biased; + _ = response.closed() => return Ok(None), + batch = input.next() => batch, + }; + let Some(batch) = next else { + break; + }; + let batch = cast_batch_to_schema(batch?, &write_schema)?; + writer + .write_arrow_batch(&batch) + .await + .map_err(to_datafusion_error)?; + } + writer + .prepare_commit() + .await + .map(Some) + .map_err(to_datafusion_error) + } + .await; + writer.close().await; + let messages = prepared.map_err(|error| { + DataFusionError::Context("no commit was attempted".into(), Box::new(error)) + })?; + let Some(messages) = messages else { + return Ok(()); + }; + if response.is_closed() { + commit.abort(&messages).await.map_err(to_datafusion_error)?; + return Ok(()); + } + // Once publication starts, finish it even if the caller cancels. Its outcome + // determines whether the prepared files can be reclaimed. + Box::pin(commit.commit(messages)).await.map_err(|error| { + DataFusionError::Context( + "publication may have succeeded".into(), + Box::new(to_datafusion_error(error)), + ) + }) +} + +fn validate_time_precision(data_type: &PaimonDataType) -> DFResult<()> { + match data_type { + PaimonDataType::Time(time) if time.precision() > 3 => { + Err(DataFusionError::NotImplemented(format!( + "CREATE TABLE AS SELECT cannot preserve TIME({}); Paimon stores TIME with millisecond precision", + time.precision() + ))) + } + PaimonDataType::Array(array) => validate_time_precision(array.element_type()), + PaimonDataType::Vector(vector) => validate_time_precision(vector.element_type()), + PaimonDataType::Multiset(multiset) => validate_time_precision(multiset.element_type()), + PaimonDataType::Map(map) => { + validate_time_precision(map.key_type())?; + validate_time_precision(map.value_type()) + } + PaimonDataType::Row(row) => row + .fields() + .iter() + .try_for_each(|field| validate_time_precision(field.data_type())), + _ => Ok(()), + } +} + +fn validate_create_table_as(ct: &CreateTable, query: &Query) -> DFResult<()> { + struct ReadOnlyQuery; + impl Visitor for ReadOnlyQuery { + type Break = (); + + fn pre_visit_statement(&mut self, statement: &Statement) -> ControlFlow<()> { + if matches!(statement, Statement::Query(_)) { + ControlFlow::Continue(()) + } else { + ControlFlow::Break(()) + } + } + + fn pre_visit_select(&mut self, select: &Select) -> ControlFlow<()> { + if select.into.is_some() { + ControlFlow::Break(()) + } else { + ControlFlow::Continue(()) + } + } + } + if let ControlFlow::Break(()) = query.visit(&mut ReadOnlyQuery) { + return Err(DataFusionError::NotImplemented( + "CREATE TABLE AS SELECT requires a read-only query".into(), + )); + } + if !ct.columns.is_empty() { + return Err(DataFusionError::NotImplemented( + "CREATE TABLE AS SELECT with explicit column declarations is not supported".into(), + )); + } + if let CreateTableOptions::Plain(options) = &ct.table_options { + if let Some(option) = options + .iter() + .find(|option| !matches!(option, SqlOption::Comment(_))) + { + return Err(DataFusionError::NotImplemented(format!( + "Unsupported CREATE TABLE AS SELECT option: {option}" + ))); + } + } + // Reject unhandled AST fields, including modifiers added by parser upgrades. + let mut supported = CreateTableBuilder::new(ct.name.clone()) + .if_not_exists(ct.if_not_exists) + .table_options(ct.table_options.clone()) + .build(); + supported.comment = ct.comment.clone(); + if ct != &supported { + return Err(DataFusionError::NotImplemented( + "Unsupported CREATE TABLE AS SELECT modifiers; only IF NOT EXISTS, PARTITIONED BY, table options and COMMENT are supported".into(), + )); + } + Ok(()) +} diff --git a/crates/integrations/datafusion/tests/create_table_as.rs b/crates/integrations/datafusion/tests/create_table_as.rs new file mode 100644 index 000000000..e4b5e5852 --- /dev/null +++ b/crates/integrations/datafusion/tests/create_table_as.rs @@ -0,0 +1,840 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod common; +#[path = "create_table_as/execution.rs"] +mod execution; +#[path = "../../../paimon/tests/mock_server.rs"] +mod mock_server; + +use std::collections::HashMap; +use std::sync::Arc; + +use arrow_array::{Array, Int32Array, Int64Array, RecordBatch}; +use arrow_schema::{DataType as ArrowType, Field, Schema as ArrowSchema}; +use common::{assert_sql_error, create_sql_context, create_test_env, exec, row_count}; +use datafusion::datasource::MemTable; +use paimon::catalog::{Catalog, Identifier, RESTCatalog}; +use paimon::spec::{BigIntType, DataType, IntType, Schema, TableType}; +use paimon::{CatalogOptions, Options}; +use paimon_datafusion::SQLContext; + +async fn query_ids(ctx: &SQLContext, sql: &str) -> Vec { + let batches = ctx.sql(sql).await.unwrap().collect().await.unwrap(); + let mut ids = batches + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .iter() + }) + .map(Option::unwrap) + .collect::>(); + ids.sort_unstable(); + ids +} + +#[tokio::test] +async fn inferred_schema_options_partitioning_and_persistence() { + let (_tmp, catalog) = create_test_env(); + let ctx = create_sql_context(catalog.clone()).await; + let source = Arc::new(ArrowSchema::new(vec![ + Field::new("id", ArrowType::Int32, false), + Field::new("part", ArrowType::Utf8, true), + ])); + let batch = RecordBatch::try_new( + source.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new(arrow_array::StringArray::from(vec![ + Some("a"), + None, + Some("b"), + ])), + ], + ) + .unwrap(); + let partitions = vec![vec![batch.slice(0, 2)], vec![batch.slice(2, 1)]]; + ctx.register_temp_table( + "source", + Arc::new(MemTable::try_new(source, partitions).unwrap()), + ) + .unwrap(); + exec( + &ctx, + "/* start */ ; ; -- empty statements + ; CREATE TABLE result PARTITIONED BY (part) + WITH ('bucket' = '-1', 'file.format' = 'parquet', 'partition.default-name' = 'it''s null') + AS SELECT id + 10 AS id, part FROM source WHERE id > 1", + ) + .await; + let table = catalog + .get_table(&Identifier::new("default", "result")) + .await + .unwrap(); + let schema = table.schema(); + assert_eq!( + schema.fields()[0].data_type(), + &DataType::BigInt(BigIntType::with_nullable(false)) + ); + assert!(schema.fields()[1].data_type().is_nullable()); + assert_eq!(schema.partition_keys(), &["part"]); + assert_eq!( + schema.options(), + &HashMap::from([ + ("bucket".into(), "-1".into()), + ("file.format".into(), "parquet".into()), + ("partition.default-name".into(), "it's null".into()), + ]), + "CTAS must preserve the supplied options without adding internal metadata" + ); + let definitions = ctx + .sql("SHOW CREATE TABLE result") + .await + .unwrap() + .collect() + .await + .unwrap(); + let definition = definitions[0] + .column(3) + .as_any() + .downcast_ref::() + .unwrap() + .value(0); + exec(&ctx, &definition.replace("\"result\"", "\"result_copy\"")).await; + exec( + &ctx, + "CREATE TABLE non_null AS WITH selected AS (SELECT part FROM source) SELECT part FROM selected WHERE part IS NOT NULL", + ) + .await; + let non_null = catalog + .get_table(&Identifier::new("default", "non_null")) + .await + .unwrap(); + assert!(non_null.schema().fields()[0].data_type().is_nullable()); + assert_eq!(row_count(&ctx, "SELECT * FROM non_null").await, 2); + let fresh = create_sql_context(catalog).await; + let rows = fresh + .sql("SELECT id, part FROM result ORDER BY id") + .await + .unwrap() + .collect() + .await + .unwrap(); + datafusion::assert_batches_eq!( + [ + "+----+------+", + "| id | part |", + "+----+------+", + "| 12 | |", + "| 13 | b |", + "+----+------+" + ], + &rows + ); +} + +#[tokio::test] +async fn empty_results_still_create_a_typed_table() { + let (_tmp, catalog) = create_test_env(); + let ctx = create_sql_context(catalog.clone()).await; + exec(&ctx, "CREATE TABLE empty COMMENT 'query result' AS SELECT CAST(NULL AS INT) AS id, CAST(1.25 AS DECIMAL(8, 2)) AS amount WHERE false").await; + let schema = catalog + .get_table(&Identifier::new("default", "empty")) + .await + .unwrap() + .schema() + .clone(); + assert_eq!(schema.fields().len(), 2); + assert_eq!(schema.comment(), Some("query result")); + assert_eq!(schema.fields()[0].name(), "id"); + assert_eq!( + schema.fields()[0].data_type(), + &DataType::Int(IntType::new()) + ); + assert_eq!(schema.fields()[1].name(), "amount"); + let DataType::Decimal(amount) = schema.fields()[1].data_type() else { + panic!("expected a decimal output"); + }; + assert_eq!((amount.precision(), amount.scale()), (8, 2)); + assert_eq!(row_count(&ctx, "SELECT * FROM empty").await, 0); +} + +#[tokio::test] +async fn time_precision_loss_is_rejected_before_table_creation() { + use arrow_array::types::Int32Type; + use arrow_array::{DictionaryArray, Time64MicrosecondArray, Time64NanosecondArray}; + + let (_tmp, catalog) = create_test_env(); + let ctx = create_sql_context(catalog.clone()).await; + let nanos = Arc::new(Time64NanosecondArray::from(vec![ + Some(45_296_123_456_789), + None, + ])); + let columns: Vec = vec![ + Arc::new(Time64MicrosecondArray::from(vec![ + Some(45_296_123_456), + None, + ])), + nanos.clone(), + Arc::new( + DictionaryArray::::try_new(Int32Array::from(vec![Some(0), None]), nanos) + .unwrap(), + ), + ]; + let schema = Arc::new(ArrowSchema::new( + ["micros", "nanos", "encoded"] + .into_iter() + .zip(&columns) + .map(|(name, column)| Field::new(name, column.data_type().clone(), true)) + .collect::>(), + )); + let batch = RecordBatch::try_new(schema.clone(), columns).unwrap(); + ctx.register_temp_table( + "time_source", + Arc::new(MemTable::try_new(schema, vec![vec![batch]]).unwrap()), + ) + .unwrap(); + for (index, (query, precision)) in [ + ("SELECT CAST('12:34:56.123456789' AS TIME) AS value", 9), + ("SELECT micros FROM time_source", 6), + ("SELECT nanos FROM time_source", 9), + ("SELECT encoded FROM time_source", 9), + ( + "SELECT named_struct('value', nanos) AS value FROM time_source", + 9, + ), + ("SELECT ARRAY[micros] AS value FROM time_source", 6), + ( + "SELECT MAP(ARRAY[1], ARRAY[nanos]) AS value FROM time_source", + 9, + ), + ( + "SELECT MAP(ARRAY[micros], ARRAY[1]) AS value FROM time_source", + 6, + ), + ( + "SELECT named_struct('values', ARRAY[encoded]) AS value FROM time_source", + 9, + ), + ("SELECT nanos FROM time_source WHERE false", 9), + ] + .iter() + .enumerate() + { + let target = format!("rejected_time_{index}"); + assert_sql_error( + &ctx, + &format!("CREATE TABLE {target} AS {query}"), + &format!("cannot preserve TIME({precision})"), + ) + .await; + assert!( + matches!( + catalog + .get_table(&Identifier::new("default", &target)) + .await, + Err(paimon::Error::TableNotExist { .. }) + ), + "{query}" + ); + } +} + +#[tokio::test] +async fn supported_time_outputs_preserve_values_and_nulls() { + use arrow_array::{Time32MillisecondArray, Time32SecondArray}; + + let (_tmp, catalog) = create_test_env(); + let ctx = create_sql_context(catalog.clone()).await; + let columns: Vec = vec![ + Arc::new(Time32SecondArray::from(vec![Some(45_296), None])), + Arc::new(Time32MillisecondArray::from(vec![Some(45_296_123), None])), + ]; + let schema = Arc::new(ArrowSchema::new( + ["seconds", "millis"] + .into_iter() + .zip(&columns) + .map(|(name, column)| Field::new(name, column.data_type().clone(), true)) + .collect::>(), + )); + let batch = RecordBatch::try_new(schema.clone(), columns).unwrap(); + ctx.register_temp_table( + "time_source", + Arc::new(MemTable::try_new(schema, vec![vec![batch]]).unwrap()), + ) + .unwrap(); + exec(&ctx, "CREATE TABLE supported_times AS SELECT seconds, millis, named_struct('value', millis) AS nested, ARRAY[seconds] AS items FROM time_source").await; + exec(&ctx, "CREATE TABLE time_text AS SELECT CAST(CAST('12:34:56.123456789' AS TIME) AS VARCHAR) AS value").await; + let fresh = create_sql_context(catalog).await; + let rows = fresh.sql("SELECT seconds, millis, nested.value, items[1] FROM supported_times ORDER BY seconds NULLS LAST") + .await.unwrap().collect().await.unwrap(); + for (index, expected) in [45_296_000, 45_296_123, 45_296_123, 45_296_000] + .into_iter() + .enumerate() + { + let values = rows + .iter() + .flat_map(|batch| { + batch + .column(index) + .as_any() + .downcast_ref::() + .unwrap() + .iter() + }) + .collect::>(); + assert_eq!(values, vec![Some(expected), None]); + } + let rows = fresh + .sql("SELECT value FROM time_text") + .await + .unwrap() + .collect() + .await + .unwrap(); + assert_eq!( + common::string_value(rows[0].column(0).as_ref(), 0), + "12:34:56.123456789" + ); +} + +#[tokio::test] +async fn quoted_names_and_cross_catalog_target_keep_the_session_source() { + let (_src, source) = create_test_env(); + let (_dst, destination) = create_test_env(); + let mut ctx = create_sql_context(source).await; + ctx.register_catalog("Other", destination.clone()) + .await + .unwrap(); + exec(&ctx, "CREATE DATABASE paimon.current_db").await; + exec(&ctx, "CREATE DATABASE \"Other\".\"TargetDb\"").await; + exec(&ctx, "USE paimon.current_db").await; + exec( + &ctx, + "CREATE TABLE \"Source.Table\" AS SELECT 7 AS \"Mixed.Id\"", + ) + .await; + exec( + &ctx, + "CREATE TABLE \"Other\".\"TargetDb\".\"Source.Table\" AS SELECT 99 AS \"Mixed.Id\"", + ) + .await; + exec( + &ctx, + r#"CREATE TABLE "Other"."TargetDb"."Result.Table" PARTITIONED BY ("Mixed.Id") + AS SELECT "Mixed.Id" FROM "Source.Table""#, + ) + .await; + assert_eq!( + query_ids( + &ctx, + r#"SELECT "Mixed.Id" FROM "Other"."TargetDb"."Result.Table""# + ) + .await, + [7] + ); + let result = destination + .get_table(&Identifier::new("TargetDb", "Result.Table")) + .await + .unwrap(); + assert_eq!(result.schema().partition_keys(), &["Mixed.Id"]); + exec( + &ctx, + "CREATE TABLE UPPER_TARGET AS SELECT version AS OF FROM (SELECT 1 AS version) AS src", + ) + .await; + assert_eq!( + row_count(&ctx, "SELECT * FROM upper_target WHERE of = 1").await, + 1 + ); + exec( + &ctx, + "CREATE TABLE \"PARTITIONED BY (id)\" AS SELECT 2 AS id", + ) + .await; + exec( + &ctx, + "CREATE TABLE quoted_source AS SELECT id FROM \"PARTITIONED BY (id)\"", + ) + .await; + assert_eq!( + row_count(&ctx, "SELECT * FROM quoted_source WHERE id = 2").await, + 1 + ); + exec( + &ctx, + "CREATE TABLE quoted_literal AS SELECT $$PARTITIONED BY (id)$$ AS value", + ) + .await; + assert_eq!( + row_count( + &ctx, + "SELECT * FROM quoted_literal WHERE value = 'PARTITIONED BY (id)'" + ) + .await, + 1 + ); + exec( + &ctx, + "SET datafusion.sql_parser.enable_ident_normalization = false", + ) + .await; + exec( + &ctx, + "CREATE TABLE Other.TargetDb.CaseTarget PARTITIONED BY (Part) AS SELECT 8 AS Part", + ) + .await; + let table = destination + .get_table(&Identifier::new("TargetDb", "CaseTarget")) + .await + .unwrap(); + assert_eq!(table.schema().partition_keys(), &["Part"]); + assert_eq!( + row_count( + &ctx, + "SELECT Part FROM Other.TargetDb.CaseTarget WHERE Part = 8" + ) + .await, + 1 + ); + for (index, selector) in ["VERSION AS OF 1", "TIMESTAMP AS OF '2099-01-01 00:00:00'"] + .iter() + .enumerate() + { + exec( + &ctx, + &format!("CREATE TABLE Other.TargetDb.History{index} AS SELECT Part FROM Other.TargetDb.CaseTarget {selector}"), + ) + .await; + assert_eq!( + row_count( + &ctx, + &format!("SELECT Part FROM Other.TargetDb.History{index} WHERE Part = 8") + ) + .await, + 1 + ); + } +} + +#[tokio::test] +async fn time_travel_source_is_materialized_as_a_persistent_table() { + let (_tmp, catalog) = create_test_env(); + let ctx = create_sql_context(catalog.clone()).await; + exec(&ctx, "CREATE TABLE \"源表.Table\" AS SELECT 1 AS id").await; + catalog + .create_tag( + &Identifier::new("default", "源表.Table"), + "base", + Some(1), + false, + ) + .await + .unwrap(); + exec(&ctx, "INSERT INTO \"源表.Table\" VALUES (2)").await; + let fresh = create_sql_context(catalog.clone()).await; + for (table, selector, expected) in [ + ("historical", "VERSION AS OF 1", vec![1]), + ("tagged", "VERSION AS OF 'base'", vec![1]), + ( + "at_timestamp", + "TIMESTAMP AS OF '2099-01-01 00:00:00'", + vec![1, 2], + ), + ] { + exec( + &ctx, + &format!("; CREATE TABLE {table} PARTITIONED BY (id) AS SELECT \"源表.Table\".id FROM \"源表.Table\" {selector}"), + ) + .await; + let target = catalog + .get_table(&Identifier::new("default", table)) + .await + .unwrap(); + assert_eq!(target.schema().partition_keys(), &["id"]); + assert_eq!( + query_ids(&fresh, &format!("SELECT * FROM {table}")).await, + expected + ); + } + assert_eq!(row_count(&fresh, "SELECT * FROM \"源表.Table\"").await, 2); + exec( + &ctx, + r#"CREATE TABLE joined AS + SELECT s1.id, $tag$source VERSION AS OF 99$tag$ AS note + FROM "源表.Table" VERSION AS /* tag */ OF 'base' AS s1 + JOIN "源表.Table" TIMESTAMP AS OF '2099-01-01 00:00:00' AS s2 ON s1.id = s2.id"#, + ) + .await; + assert_eq!( + query_ids( + &fresh, + "SELECT id FROM joined WHERE note = 'source VERSION AS OF 99'" + ) + .await, + [1] + ); + exec(&ctx, "CREATE DATABASE db2").await; + exec(&ctx, "CREATE TABLE db2.src AS SELECT 10 AS id").await; + exec(&ctx, "CREATE TABLE src AS SELECT 12 AS id").await; + for (table, query) in [ + ("current_join", "SELECT paimon.default.src.id AS a, db2.src.id AS b FROM paimon.default.src CROSS JOIN db2.src"), + ("historical_join", "SELECT paimon.default.src.id AS a, db2.src.id AS b FROM paimon.default.src VERSION AS OF 1 CROSS JOIN db2.src VERSION AS OF 1"), + ("aliased_join", "SELECT a.id AS a, b.id AS b FROM paimon.default.src VERSION AS OF 1 AS a CROSS JOIN db2.src VERSION AS OF 1 AS b"), + ] { + exec(&ctx, &format!("CREATE TABLE {table} AS {query}")).await; + assert_eq!(row_count(&fresh, &format!("SELECT * FROM {table} WHERE a = 12 AND b = 10")).await, 1); + } +} + +#[tokio::test] +async fn inferred_nested_types_and_repeated_source_field_ids() { + let (_tmp, catalog) = create_test_env(); + let ctx = create_sql_context(catalog.clone()).await; + exec( + &ctx, + "CREATE TABLE source AS + SELECT named_struct('x', 1, 'text', 'hello') AS item, ARRAY[1, 2] AS items, + MAP(ARRAY['key'], ARRAY[3]) AS entries, CAST(1.25 AS DECIMAL(8, 2)) AS amount, + TIMESTAMP '2026-11-17 01:02:03' AS ts", + ) + .await; + exec( + &ctx, + "CREATE TABLE copied AS SELECT item AS first, item AS second, items, entries, amount, ts FROM source", + ) + .await; + let schema = catalog + .get_table(&Identifier::new("default", "copied")) + .await + .unwrap() + .schema() + .clone(); + assert_ne!(schema.fields()[0].id(), schema.fields()[1].id()); + if let (DataType::Row(first), DataType::Row(second)) = ( + schema.fields()[0].data_type(), + schema.fields()[1].data_type(), + ) { + assert_ne!(first.fields()[0].id(), second.fields()[0].id()); + } else { + panic!("expected nested ROW types"); + } + let rows = row_count( + &ctx, + "SELECT * FROM copied WHERE first.x = 1 AND second.text = 'hello' AND items[1] = 1 + AND entries['key'] = 3 AND amount = 1.25 AND ts = TIMESTAMP '2026-11-17 01:02:03'", + ) + .await; + assert_eq!(rows, 1); +} + +#[tokio::test] +async fn inferred_geospatial_types_preserve_crs_and_algorithm() { + let (_tmp, catalog) = create_test_env(); + let ctx = create_sql_context(catalog.clone()).await; + let wkb = "decode('0101000000000000000000F03F0000000000000040', 'hex')"; + for (name, predicate, expected_rows) in [("shapes", "true", 1), ("empty_shapes", "false", 0)] { + exec( + &ctx, + &format!( + "CREATE TABLE {name} AS SELECT + CAST({wkb} AS GEOMETRY('EPSG:3857')) AS geom, + CAST({wkb} AS GEOGRAPHY('OGC:CRS84', karney)) AS geog + WHERE {predicate}" + ), + ) + .await; + let table = catalog + .get_table(&Identifier::new("default", name)) + .await + .unwrap(); + assert!(matches!( + table.schema().fields()[0].data_type(), + DataType::Geometry(geometry) if geometry.crs() == "EPSG:3857" + )); + assert!(matches!( + table.schema().fields()[1].data_type(), + DataType::Geography(geography) + if geography.crs() == "OGC:CRS84" && geography.algorithm().to_string() == "karney" + )); + assert_eq!( + row_count( + &ctx, + &format!("SELECT * FROM {name} WHERE geom = {wkb} AND geog = {wkb}") + ) + .await, + expected_rows + ); + } +} + +#[tokio::test] +async fn arrow_layouts_are_normalized_for_scalar_and_nested_outputs() { + use arrow_array::builder::StringDictionaryBuilder; + use arrow_array::types::Int32Type; + use arrow_array::LargeStringArray; + + let (_tmp, catalog) = create_test_env(); + let ctx = create_sql_context(catalog).await; + let mut dictionary = StringDictionaryBuilder::::new(); + dictionary.append("encoded").unwrap(); + dictionary.append_null(); + let columns: Vec = vec![ + Arc::new(dictionary.finish()), + Arc::new(LargeStringArray::from(vec![Some("large"), None])), + ]; + let schema = Arc::new(ArrowSchema::new(vec![ + Field::new("encoded", columns[0].data_type().clone(), true), + Field::new("large", ArrowType::LargeUtf8, true), + ])); + let batch = RecordBatch::try_new(schema.clone(), columns).unwrap(); + ctx.register_temp_table( + "source", + Arc::new(MemTable::try_new(schema, vec![vec![batch]]).unwrap()), + ) + .unwrap(); + exec(&ctx, "CREATE TABLE decoded AS SELECT *, named_struct('encoded', encoded, 'large', large) AS nested, ARRAY[large] AS items FROM source").await; + assert_eq!( + row_count( + &ctx, + "SELECT * FROM decoded WHERE encoded = 'encoded' AND large = 'large' AND nested.encoded = 'encoded' AND nested.large = 'large' AND items[1] = 'large'" + ) + .await, + 1 + ); + assert_eq!( + row_count( + &ctx, + "SELECT * FROM decoded WHERE encoded IS NULL AND large IS NULL AND nested.encoded IS NULL AND nested.large IS NULL AND items[1] IS NULL" + ) + .await, + 1 + ); + use arrow_array::builder::{Int64Builder, MapBuilder, MapFieldNames, StringBuilder}; + use arrow_array::MapArray; + let names = MapFieldNames { + entry: "entries".into(), + key: "key".into(), + value: "value".into(), + }; + let mut map = MapBuilder::new(Some(names), StringBuilder::new(), Int64Builder::new()); + map.keys().append_value("discarded"); + map.values().append_value(0); + map.append(true).unwrap(); + map.keys().append_value("key"); + map.values().append_value(9); + map.append(true).unwrap(); + map.append(false).unwrap(); + let (field, offsets, entries, nulls, _) = map.finish().into_parts(); + for sorted in [false, true] { + let map = MapArray::new( + field.clone(), + offsets.clone(), + entries.clone(), + nulls.clone(), + sorted, + ) + .slice(1, 2); + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + "m", + map.data_type().clone(), + true, + )])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(map)]).unwrap(); + ctx.register_temp_table( + format!("maps_{sorted}"), + Arc::new(MemTable::try_new(schema, vec![vec![batch]]).unwrap()), + ) + .unwrap(); + exec(&ctx, &format!("CREATE TABLE map_copy_{sorted} AS SELECT m, named_struct('m', m) AS nested, ARRAY[m] AS items FROM maps_{sorted}")).await; + assert_eq!(row_count(&ctx, &format!("SELECT * FROM map_copy_{sorted} WHERE m['key'] = 9 AND nested.m['key'] = 9 AND items[1]['key'] = 9")).await, 1); + assert_eq!(row_count(&ctx, &format!("SELECT * FROM map_copy_{sorted} WHERE m IS NULL AND nested.m IS NULL AND items[1] IS NULL")).await, 1); + } +} + +#[tokio::test] +async fn existing_target_is_a_noop_without_source_planning_or_execution() { + let (_tmp, catalog) = create_test_env(); + let ctx = create_sql_context(catalog.clone()).await; + exec(&ctx, "CREATE TABLE existing AS SELECT 9 AS id").await; + for sql in [ + "; CREATE TABLE IF NOT EXISTS existing AS SELECT missing_function(id) FROM missing_source", + "; CREATE TABLE IF NOT EXISTS existing AS SELECT * FROM missing_source VERSION AS OF 1", + "; CREATE TABLE IF NOT EXISTS existing AS SELECT * FROM missing_source TIMESTAMP AS OF '2099-01-01 00:00:00'", + ] { + exec(&ctx, sql).await; + } + assert_sql_error( + &ctx, + "CREATE TABLE existing AS SELECT * FROM missing_source", + "already exists", + ) + .await; + assert_eq!(query_ids(&ctx, "SELECT * FROM existing").await, [9]); + catalog + .create_table( + &Identifier::new("default", "external_target"), + Schema::builder() + .column("id", DataType::Int(IntType::new())) + .option("type", TableType::IcebergTable.as_str()) + .build() + .unwrap(), + false, + ) + .await + .unwrap(); + exec( + &ctx, + "CREATE TABLE IF NOT EXISTS external_target AS SELECT * FROM missing_source", + ) + .await; +} + +#[tokio::test] +async fn invalid_ctas_preserves_catalog_and_source_data() { + let (_tmp, catalog) = create_test_env(); + let ctx = create_sql_context(catalog.clone()).await; + exec(&ctx, "CREATE TABLE source AS SELECT 1 AS id").await; + ctx.register_temp_table( + "empty_columns", + Arc::new(MemTable::try_new(Arc::new(ArrowSchema::empty()), vec![vec![]]).unwrap()), + ) + .unwrap(); + for sql in [ + "CREATE TABLE bad AUTO_INCREMENT = 42 AS SELECT 1 AS id", + "CREATE TABLE bad DEFAULT CHARSET = utf8mb4 AS SELECT 1 AS id", + "CREATE TABLE bad COLLATE = utf8mb4_bin AS SELECT 1 AS id", + "CREATE TABLE bad WITH ('type' = 'iceberg-table') TBLPROPERTIES ('bucket' = '-1') AS SELECT 1 AS id", + "CREATE TABLE bad WITH ('bucket' = '0') OPTIONS ('file.format' = 'parquet') AS SELECT 1 AS id", + "CREATE TABLE bad WITH () OPTIONS ('bucket' = '-1') AS SELECT 1 AS id", + "CREATE TABLE bad AS SELECT * FROM empty_columns", + "CREATE TABLE bad AS SELECT NULL AS value", + "CREATE TABLE bad AS SELECT 1 AS id, 2 AS id", + "CREATE TABLE bad AS SELECT 1 AS \"_KEY_id\"", + "CREATE TABLE bad AS SELECT 1 AS alias VERSION AS OF 1", + "CREATE TABLE bad PARTITIONED BY (missing) AS SELECT 1 AS id", + r#"CREATE TABLE bad PARTITIONED BY (s.id) AS SELECT 1 AS "s.id""#, + "CREATE TABLE bad AS SELECT 1 AS id PARTITIONED BY (id)", + "CREATE TABLE bad WITH ('bucket' = '0') AS SELECT 1 AS id", + "CREATE EXTERNAL TABLE bad AS SELECT 1 AS id", + "CREATE TRANSIENT TABLE bad AS SELECT 1 AS id", + "CREATE GLOBAL TABLE bad AS SELECT 1 AS id", + "CREATE TABLE bad LOCATION '/tmp/not-a-ctas-table' AS SELECT 1 AS id", + "CREATE TABLE bad WITH ('type' = 'iceberg-table') AS SELECT 1 AS id", + "CREATE TABLE bad WITH ('type' = 'format-table') AS SELECT 1 AS id", + "CREATE TABLE bad (PRIMARY KEY (id)) AS SELECT 1 AS id", + ] { + assert!(ctx.sql(sql).await.is_err(), "{sql}"); + } + for (statement, expected_error) in [ + ( + "CREATE OR REPLACE TABLE source AS SELECT 2 AS id", + "does not support modifiers before TABLE", + ), + ( + "CREATE OR REPLACE TABLE source PARTITIONED BY (id) AS SELECT * FROM missing_source VERSION AS OF 1", + "does not support modifiers before TABLE", + ), + ( + "CREATE OR ALTER TABLE source AS SELECT * FROM source TIMESTAMP AS OF '2099-01-01 00:00:00'", + "does not support modifiers before TABLE", + ), + ( + "CREATE TABLE source (id BIGINT) AS SELECT * FROM source VERSION AS OF 1", + "explicit column declarations", + ), + ( + "CREATE TABLE bad AS SELECT * FROM source VERSION AS OF 1; DROP TABLE source", + "Expected exactly one SQL statement", + ), + ] { + assert_sql_error(&ctx, &format!("/* start */ ; ; {statement}"), expected_error).await; + } + for sql in [ + "; CREATE TABLE bad AS SELECT id INTO side_effect FROM source VERSION AS OF 1", + "CREATE TABLE bad AS WITH changed AS (SELECT id INTO side_effect FROM source) SELECT * FROM changed", + "CREATE TABLE bad AS SELECT id FROM source UNION ALL SELECT id INTO side_effect FROM source", + "CREATE TABLE bad AS INSERT INTO source VALUES (2)", + "CREATE TABLE bad AS UPDATE source SET id = 2", + "CREATE TABLE bad AS DELETE FROM source", + "CREATE TABLE bad AS MERGE INTO source t USING (SELECT 1 AS id) s ON t.id = s.id WHEN MATCHED THEN DELETE", + "CREATE TABLE bad AS WITH changed AS (INSERT INTO source VALUES (2) RETURNING id) SELECT * FROM changed", + ] { + assert_sql_error(&ctx, sql, "requires a read-only query").await; + } + for missing in ["bad", "side_effect"] { + assert!(matches!( + catalog + .get_table(&Identifier::new("default", missing)) + .await, + Err(paimon::Error::TableNotExist { .. }) + )); + } + assert_eq!(query_ids(&ctx, "SELECT id FROM source").await, [1]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn rest_sql_function_uses_session_namespace_for_persistent_targets() { + let tmp = tempfile::tempdir().unwrap(); + let server = mock_server::start_mock_server( + "ctas".into(), + tmp.path().to_string_lossy().into_owned(), + paimon::api::ConfigResponse::new(HashMap::from([( + CatalogOptions::PREFIX.to_string(), + "mock-test".into(), + )])), + vec!["source_db".into(), "default".into()], + ) + .await; + let mut options = Options::new(); + options.set(CatalogOptions::URI, server.url().unwrap()); + options.set(CatalogOptions::WAREHOUSE, "ctas"); + options.set(CatalogOptions::TOKEN_PROVIDER, "bear"); + options.set(CatalogOptions::TOKEN, "test-token"); + let source = Arc::new(RESTCatalog::new(options, true).await.unwrap()); + let (_dest, destination) = create_test_env(); + let mut ctx = SQLContext::new(); + ctx.register_catalog("source", source).await.unwrap(); + ctx.register_catalog("dest", destination).await.unwrap(); + exec(&ctx, "USE source.source_db").await; + exec( + &ctx, + "CREATE FUNCTION plus_one(x BIGINT) RETURNS BIGINT RETURN x + 1", + ) + .await; + exec(&ctx, "CREATE TABLE inputs AS SELECT 8 AS id").await; + for target in ["dest.default.result", "source.default.result"] { + let before = server.table_get_call_count("source_db", "inputs"); + exec( + &ctx, + &format!("CREATE TABLE {target} WITH ('commit.max-retries' = '0') AS SELECT plus_one(id) AS id FROM inputs"), + ) + .await; + assert_eq!( + server.table_get_call_count("source_db", "inputs") - before, + 1 + ); + assert_eq!( + query_ids(&ctx, &format!("SELECT * FROM {target}")).await, + [9] + ); + } +} diff --git a/crates/integrations/datafusion/tests/create_table_as/execution.rs b/crates/integrations/datafusion/tests/create_table_as/execution.rs new file mode 100644 index 000000000..62fcd37c4 --- /dev/null +++ b/crates/integrations/datafusion/tests/create_table_as/execution.rs @@ -0,0 +1,716 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use async_trait::async_trait; +use bytes::Bytes; +use datafusion::catalog::streaming::StreamingTable; +use datafusion::catalog::Session; +use datafusion::datasource::{TableProvider, TableType as DFTableType}; +use datafusion::error::{DataFusionError, Result as DFResult}; +use datafusion::execution::{SendableRecordBatchStream, TaskContext}; +use datafusion::logical_expr::{create_udf, ColumnarValue, Expr, Volatility}; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::streaming::{PartitionStream, StreamingTableExec}; +use datafusion::physical_plan::ExecutionPlan; +use futures::StreamExt; +use paimon::catalog::Database; +use paimon::io::{FileIOBuilder, FileIOProvider}; +use paimon::spec::SchemaChange; +use paimon::table::Table; +use paimon::FileSystemCatalog; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::time::Duration; +use tokio::sync::Notify; + +#[derive(Clone, Copy, Debug, PartialEq)] +enum Fault { + None, + Lookup, + Create, + Race, + LoadAfterCreate, + Write, + WriteAfterOutput, + BeforePublication, + LostResponse, + PausePublication, +} + +#[derive(Debug)] +struct InjectedIO { + operator: opendal::Operator, + fault: Fault, + published: AtomicBool, + data_paths: std::sync::Mutex>, + reached: Notify, + resume: Notify, +} + +fn injected_error(message: &str) -> paimon::Error { + paimon::Error::UnexpectedError { + message: message.into(), + source: None, + } +} + +fn relative(path: &str) -> &str { + path.trim_start_matches("file://").trim_start_matches('/') +} + +#[async_trait] +impl FileIOProvider for InjectedIO { + async fn create(&self, path: &str) -> paimon::Result<(opendal::Operator, String)> { + if path.ends_with(".parquet") { + let mut paths = self.data_paths.lock().unwrap(); + let index = paths + .iter() + .position(|entry| entry == path) + .unwrap_or_else(|| { + paths.push(path.into()); + paths.len() - 1 + }); + if self.fault == Fault::Write || self.fault == Fault::WriteAfterOutput && index == 1 { + return Err(injected_error("injected data write failure")); + } + } + if self.published.load(Ordering::SeqCst) + && self.fault == Fault::LostResponse + && path.contains("/snapshot/") + { + return Err(injected_error("injected snapshot read failure")); + } + Ok((self.operator.clone(), relative(path).into())) + } + + async fn write_atomic_if_not_exists( + &self, + path: &str, + bytes: Bytes, + ) -> paimon::Result> { + if path.contains("/snapshot/snapshot-") { + if self.fault == Fault::PausePublication { + self.reached.notify_one(); + self.resume.notified().await; + self.operator.write(relative(path), bytes).await.unwrap(); + self.published.store(true, Ordering::SeqCst); + return Ok(Some(true)); + } + if self.fault == Fault::BeforePublication { + return Err(injected_error("injected publication failure")); + } + if self.fault == Fault::LostResponse { + self.operator.write(relative(path), bytes).await.unwrap(); + self.published.store(true, Ordering::SeqCst); + return Err(injected_error("injected lost commit response")); + } + } + Ok(None) + } +} + +#[derive(Debug)] +struct TestCatalog { + inner: Arc, + fault: Fault, + creates: AtomicUsize, + drops: AtomicUsize, + io: Option>, +} + +impl TestCatalog { + fn new(inner: Arc, fault: Fault) -> Arc { + let io = matches!( + fault, + Fault::Write + | Fault::WriteAfterOutput + | Fault::BeforePublication + | Fault::LostResponse + | Fault::PausePublication + ) + .then(|| { + Arc::new(InjectedIO { + operator: + opendal::Operator::from_config(opendal::services::MemoryConfig::default()) + .unwrap(), + fault, + published: AtomicBool::new(false), + data_paths: std::sync::Mutex::new(Vec::new()), + reached: Notify::new(), + resume: Notify::new(), + }) + }); + Arc::new(Self { + inner, + fault, + creates: AtomicUsize::new(0), + drops: AtomicUsize::new(0), + io, + }) + } +} + +async fn append(table: &Table, value: i64) { + let builder = table.new_write_builder(); + let mut writer = builder.new_write().unwrap(); + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + "id", + ArrowType::Int64, + false, + )])); + writer + .write_arrow_batch( + &RecordBatch::try_new(schema, vec![Arc::new(Int64Array::from(vec![value]))]).unwrap(), + ) + .await + .unwrap(); + let messages = writer.prepare_commit().await.unwrap(); + builder.new_commit().commit(messages).await.unwrap(); +} + +#[async_trait] +impl Catalog for TestCatalog { + async fn list_databases(&self) -> paimon::Result> { + self.inner.list_databases().await + } + async fn create_database( + &self, + name: &str, + ignore: bool, + properties: HashMap, + ) -> paimon::Result<()> { + self.inner.create_database(name, ignore, properties).await + } + async fn get_database(&self, name: &str) -> paimon::Result { + self.inner.get_database(name).await + } + async fn drop_database(&self, name: &str, ignore: bool, cascade: bool) -> paimon::Result<()> { + self.inner.drop_database(name, ignore, cascade).await + } + async fn get_table(&self, id: &Identifier) -> paimon::Result { + if self.fault == Fault::Lookup { + return Err(injected_error("injected forbidden lookup")); + } + if self.fault == Fault::LoadAfterCreate && self.creates.load(Ordering::SeqCst) > 0 { + return Err(injected_error("injected table load failure")); + } + let table = self.inner.get_table(id).await?; + if let Some(io) = &self.io { + let schema_path = format!("{}/schema/schema-0", table.location()); + if !io.operator.exists(relative(&schema_path)).await.unwrap() { + io.operator + .write( + relative(&schema_path), + serde_json::to_vec(table.schema()).unwrap(), + ) + .await + .unwrap(); + } + Ok(Table::new( + FileIOBuilder::new("memory") + .with_provider(io.clone()) + .build()?, + id.clone(), + table.location().to_string(), + table.schema().clone(), + None, + )) + } else { + Ok(table) + } + } + async fn list_tables(&self, db: &str) -> paimon::Result> { + self.inner.list_tables(db).await + } + async fn create_table( + &self, + id: &Identifier, + schema: Schema, + ignore: bool, + ) -> paimon::Result<()> { + self.creates.fetch_add(1, Ordering::SeqCst); + assert!( + !ignore, + "CTAS must use strict creation even with IF NOT EXISTS" + ); + if self.fault == Fault::Create { + return Err(injected_error("injected table creation failure")); + } + if self.fault == Fault::Race { + self.inner.create_table(id, schema.clone(), false).await?; + append(&self.inner.get_table(id).await?, 77).await; + } + self.inner.create_table(id, schema, ignore).await + } + async fn drop_table(&self, id: &Identifier, ignore: bool) -> paimon::Result<()> { + self.drops.fetch_add(1, Ordering::SeqCst); + self.inner.drop_table(id, ignore).await + } + async fn rename_table( + &self, + from: &Identifier, + to: &Identifier, + ignore: bool, + ) -> paimon::Result<()> { + self.inner.rename_table(from, to, ignore).await + } + async fn alter_table( + &self, + id: &Identifier, + changes: Vec, + ignore: bool, + ) -> paimon::Result<()> { + self.inner.alter_table(id, changes, ignore).await + } +} + +#[derive(Debug)] +struct ProbeSource { + schema: Arc, + scans: AtomicUsize, + executions: Arc, + fail: bool, + concurrent_writer: Option>, +} +impl ProbeSource { + fn new(fail: bool, concurrent_writer: Option>) -> Arc { + Arc::new(Self { + schema: Arc::new(ArrowSchema::new(vec![Field::new( + "id", + ArrowType::Int64, + false, + )])), + scans: AtomicUsize::new(0), + executions: Arc::new(AtomicUsize::new(0)), + fail, + concurrent_writer, + }) + } +} +#[derive(Debug)] +struct ProbePartition { + schema: Arc, + executions: Arc, + fail: bool, + concurrent_writer: Option>, +} +impl PartitionStream for ProbePartition { + fn schema(&self) -> &Arc { + &self.schema + } + fn execute(&self, _: Arc) -> SendableRecordBatchStream { + self.executions.fetch_add(1, Ordering::SeqCst); + let first = RecordBatch::try_new( + self.schema.clone(), + vec![Arc::new(Int64Array::from(vec![11]))], + ) + .unwrap(); + let concurrent_writer = self.concurrent_writer.clone(); + let fail = self.fail; + let schema = self.schema.clone(); + let stream = + futures::stream::once(async { Ok(first) }).chain(futures::stream::once(async move { + if let Some(catalog) = concurrent_writer { + let table = catalog + .get_table(&Identifier::new("default", "target")) + .await + .unwrap(); + append(&table, 88).await; + } + if fail { + Err(DataFusionError::Execution("injected source failure".into())) + } else { + Ok(RecordBatch::try_new( + schema, + vec![Arc::new(Int64Array::from(vec![12]))], + )?) + } + })); + Box::pin(RecordBatchStreamAdapter::new(self.schema.clone(), stream)) + } +} +#[async_trait] +impl TableProvider for ProbeSource { + fn schema(&self) -> Arc { + self.schema.clone() + } + fn table_type(&self) -> DFTableType { + DFTableType::Base + } + async fn scan( + &self, + _: &dyn Session, + projection: Option<&Vec>, + _: &[Expr], + limit: Option, + ) -> DFResult> { + self.scans.fetch_add(1, Ordering::SeqCst); + Ok(Arc::new(StreamingTableExec::try_new( + self.schema.clone(), + vec![Arc::new(ProbePartition { + schema: self.schema.clone(), + executions: self.executions.clone(), + fail: self.fail, + concurrent_writer: self.concurrent_writer.clone(), + })], + projection, + [], + false, + limit, + )?)) + } +} + +async fn context(catalog: Arc, source: Arc) -> SQLContext { + let mut ctx = SQLContext::new(); + ctx.register_catalog("paimon", catalog).await.unwrap(); + ctx.register_temp_table("source", source).unwrap(); + ctx +} + +#[tokio::test] +async fn one_source_plan_is_reused_for_inference_and_execution() { + let (_tmp, inner) = create_test_env(); + let catalog = TestCatalog::new(inner, Fault::None); + let source = ProbeSource::new(false, None); + let ctx = context(catalog.clone(), source.clone()).await; + let observed_rows = Arc::new(AtomicUsize::new(0)); + let counter = observed_rows.clone(); + ctx.ctx().register_udf(create_udf( + "observe", + vec![ArrowType::Int64], + ArrowType::Int64, + Volatility::Volatile, + Arc::new(move |args| { + let ColumnarValue::Array(ids) = &args[0] else { + panic!("expected the source column"); + }; + counter.fetch_add(ids.len(), Ordering::SeqCst); + Ok(args[0].clone()) + }), + )); + let query = "SELECT observe(id) AS id FROM source"; + exec(&ctx, &format!("CREATE TABLE target AS {query}")).await; + assert_eq!(source.scans.load(Ordering::SeqCst), 1); + assert_eq!(source.executions.load(Ordering::SeqCst), 1); + assert_eq!(query_ids(&ctx, "SELECT * FROM target").await, [11, 12]); + assert_eq!(catalog.creates.load(Ordering::SeqCst), 1); + exec( + &ctx, + &format!("CREATE TABLE IF NOT EXISTS target AS {query}"), + ) + .await; + assert_eq!(source.scans.load(Ordering::SeqCst), 1); + assert_eq!(source.executions.load(Ordering::SeqCst), 1); + assert_eq!(observed_rows.load(Ordering::SeqCst), 2); + assert_eq!(catalog.creates.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn create_race_never_populates_or_drops_the_winner() { + for if_not_exists in [false, true] { + let (_tmp, inner) = create_test_env(); + let catalog = TestCatalog::new(inner.clone(), Fault::Race); + let source = ProbeSource::new(false, None); + let ctx = context(catalog.clone(), source.clone()).await; + let clause = if if_not_exists { "IF NOT EXISTS " } else { "" }; + let result = ctx + .sql(&format!( + "CREATE TABLE {clause}target AS SELECT * FROM source" + )) + .await; + assert_eq!(result.is_ok(), if_not_exists); + if let Err(error) = result { + assert!(error.to_string().contains("already exists"), "{error}"); + } + assert_eq!(catalog.creates.load(Ordering::SeqCst), 1); + assert_eq!(source.executions.load(Ordering::SeqCst), 0); + assert_eq!(catalog.drops.load(Ordering::SeqCst), 0); + assert_eq!(query_ids(&ctx, "SELECT * FROM target").await, [77]); + } +} + +#[tokio::test] +async fn catalog_failures_do_not_execute_the_source_or_drop_the_target() { + for (fault, expected_error) in [ + (Fault::Lookup, "injected forbidden lookup"), + (Fault::Create, "injected table creation failure"), + (Fault::LoadAfterCreate, "injected table load failure"), + ] { + for if_not_exists in [false, true] { + let (_tmp, inner) = create_test_env(); + let catalog = TestCatalog::new(inner.clone(), fault); + let source = ProbeSource::new(false, None); + let ctx = context(catalog.clone(), source.clone()).await; + let clause = if if_not_exists { "IF NOT EXISTS " } else { "" }; + assert_sql_error( + &ctx, + &format!("CREATE TABLE {clause}target AS SELECT * FROM source"), + expected_error, + ) + .await; + let planned = usize::from(fault != Fault::Lookup); + assert_eq!(catalog.creates.load(Ordering::SeqCst), planned); + assert_eq!(source.scans.load(Ordering::SeqCst), planned); + assert_eq!(source.executions.load(Ordering::SeqCst), 0); + assert_eq!(catalog.drops.load(Ordering::SeqCst), 0); + let target = inner.get_table(&Identifier::new("default", "target")).await; + if fault == Fault::LoadAfterCreate { + assert!(target + .unwrap() + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .is_none()); + } else { + assert!(matches!(target, Err(paimon::Error::TableNotExist { .. }))); + } + } + } +} + +#[tokio::test] +async fn population_preserves_concurrent_rows_and_cleans_failed_output() { + for (fail, concurrent) in [(true, false), (true, true), (false, true)] { + let (tmp, inner) = create_test_env(); + let catalog = TestCatalog::new(inner.clone(), Fault::None); + let source = ProbeSource::new(fail, concurrent.then_some(inner)); + let ctx = context(catalog.clone(), source.clone()).await; + let sql = "CREATE TABLE target WITH ('target-file-size' = '1 b') AS SELECT * FROM source"; + if fail { + assert_sql_error(&ctx, sql, "injected source failure").await; + } else { + exec(&ctx, sql).await; + } + assert_eq!(catalog.drops.load(Ordering::SeqCst), 0); + let mut expected = if fail { vec![] } else { vec![11, 12] }; + if concurrent { + expected.push(88); + } + assert_eq!(query_ids(&ctx, "SELECT * FROM target").await, expected); + let data_files = common::files_under(tmp.path()) + .iter() + .filter(|path| path.extension().is_some_and(|ext| ext == "parquet")) + .count(); + assert_eq!( + data_files, + expected.len(), + "only committed output should remain: fail={fail}, concurrent={concurrent}" + ); + exec( + &ctx, + "CREATE TABLE IF NOT EXISTS target AS SELECT * FROM source", + ) + .await; + assert_eq!(catalog.creates.load(Ordering::SeqCst), 1); + assert_eq!(source.executions.load(Ordering::SeqCst), 1); + } +} + +#[tokio::test] +async fn write_and_publication_failures_keep_the_table_and_uncertain_files() { + for fault in [ + Fault::Write, + Fault::WriteAfterOutput, + Fault::BeforePublication, + Fault::LostResponse, + ] { + let (_tmp, inner) = create_test_env(); + let catalog = TestCatalog::new(inner, fault); + let source = ProbeSource::new(false, None); + let ctx = context(catalog.clone(), source.clone()).await; + let error = ctx + .sql("CREATE TABLE target WITH ('target-file-size' = '1 b', 'commit.max-retries' = '0') AS SELECT * FROM source") + .await + .expect_err("injected failure") + .to_string(); + assert!(error.contains("injected"), "{fault:?}: {error}"); + assert_eq!(catalog.drops.load(Ordering::SeqCst), 0); + catalog + .inner + .get_table(&Identifier::new("default", "target")) + .await + .unwrap(); + let io = catalog.io.as_ref().unwrap(); + let entries = io.operator.list_with("/").recursive(true).await.unwrap(); + let data = entries + .iter() + .filter(|entry| entry.path().ends_with(".parquet")) + .count(); + if matches!(fault, Fault::Write | Fault::WriteAfterOutput) { + assert_eq!(data, 0); + assert_eq!( + io.data_paths.lock().unwrap().len(), + if fault == Fault::Write { 1 } else { 2 } + ); + assert!(error.contains("no commit was attempted"), "{error}"); + } else { + assert!( + data > 0, + "prepared files must survive uncertain publication" + ); + assert!(error.contains("publication may have succeeded"), "{error}"); + } + assert_eq!( + io.published.load(Ordering::SeqCst), + fault == Fault::LostResponse + ); + exec( + &ctx, + "CREATE TABLE IF NOT EXISTS target AS SELECT * FROM source", + ) + .await; + assert_eq!(catalog.creates.load(Ordering::SeqCst), 1); + assert_eq!(source.executions.load(Ordering::SeqCst), 1); + if fault == Fault::LostResponse { + io.published.store(false, Ordering::SeqCst); + assert_eq!(query_ids(&ctx, "SELECT * FROM target").await, [11, 12]); + } + } +} + +#[derive(Debug)] +struct PausedSource { + schema: Arc, + reached: Arc, + resume: Arc, + early_rows: usize, +} + +impl PartitionStream for PausedSource { + fn schema(&self) -> &Arc { + &self.schema + } + fn execute(&self, _: Arc) -> SendableRecordBatchStream { + let batch = |value, rows| { + RecordBatch::try_new( + self.schema.clone(), + vec![Arc::new(Int64Array::from(vec![value; rows]))], + ) + .unwrap() + }; + let first = batch(12, self.early_rows); + let second = batch(11, 1); + let reached = self.reached.clone(); + let resume = self.resume.clone(); + let input = futures::stream::iter([Ok(first)]).chain(futures::stream::once(async move { + reached.notify_one(); + resume.notified().await; + Ok(second) + })); + Box::pin(RecordBatchStreamAdapter::new(self.schema.clone(), input)) + } +} + +fn paused_source(ctx: &SQLContext, early_rows: usize) -> (Arc, Arc) { + let schema = Arc::new(ArrowSchema::new(vec![Field::new( + "id", + ArrowType::Int64, + false, + )])); + let reached = Arc::new(Notify::new()); + let resume = Arc::new(Notify::new()); + let source = PausedSource { + schema: schema.clone(), + reached: reached.clone(), + resume: resume.clone(), + early_rows, + }; + let table = StreamingTable::try_new(schema, vec![Arc::new(source)]).unwrap(); + ctx.register_temp_table("probe", Arc::new(table)).unwrap(); + (reached, resume) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cancellation_discards_uncommitted_output() { + let (tmp, catalog) = create_test_env(); + let ctx = create_sql_context(catalog.clone()).await; + let (_reached, _resume) = paused_source(&ctx, 24576); + let statement = tokio::spawn(async move { + ctx.sql("CREATE TABLE result WITH ('target-file-size' = '1 b') AS SELECT id FROM probe") + .await + .map(|_| ()) + }); + let outputs = || { + common::files_under(tmp.path()) + .into_iter() + .filter(|p| p.extension().is_some_and(|e| e == "parquet")) + .collect::>() + }; + tokio::time::timeout(Duration::from_secs(10), async { + while !outputs() + .iter() + .any(|path| path.metadata().is_ok_and(|metadata| metadata.len() > 0)) + { + tokio::time::sleep(Duration::from_millis(15)).await; + } + }) + .await + .unwrap(); + statement.abort(); + assert!(statement.await.unwrap_err().is_cancelled()); + tokio::time::timeout(Duration::from_secs(10), async { + while !outputs().is_empty() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert!(outputs().is_empty(), "orphan files: {:?}", outputs()); + let table = catalog + .get_table(&Identifier::new("default", "result")) + .await + .unwrap(); + assert!(table + .snapshot_manager() + .get_latest_snapshot() + .await + .unwrap() + .is_none()); + let fresh = create_sql_context(catalog).await; + assert_eq!(row_count(&fresh, "SELECT * FROM result").await, 0); +} + +#[tokio::test] +async fn cancellation_finishes_started_publication() { + let (_tmp, inner) = create_test_env(); + let catalog = TestCatalog::new(inner, Fault::PausePublication); + let ctx = Arc::new(context(catalog.clone(), ProbeSource::new(false, None)).await); + let writer_ctx = ctx.clone(); + let statement = tokio::spawn(async move { + writer_ctx + .sql("CREATE TABLE target AS SELECT * FROM source") + .await + .map(|_| ()) + }); + let io = catalog.io.as_ref().unwrap(); + tokio::time::timeout(Duration::from_secs(10), io.reached.notified()) + .await + .unwrap(); + statement.abort(); + assert!(statement.await.unwrap_err().is_cancelled()); + io.resume.notify_one(); + tokio::time::timeout(Duration::from_secs(10), async { + while !io.published.load(Ordering::SeqCst) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert_eq!(query_ids(&ctx, "SELECT id FROM target").await, [11, 12]); +} diff --git a/crates/integrations/datafusion/tests/read_files_behavior.rs b/crates/integrations/datafusion/tests/read_files_behavior.rs index 6cba4097c..c4f9c16da 100644 --- a/crates/integrations/datafusion/tests/read_files_behavior.rs +++ b/crates/integrations/datafusion/tests/read_files_behavior.rs @@ -291,10 +291,15 @@ async fn wildcard_replacements_preserve_aliases_for_outer_queries_and_ctas() { ) .await .unwrap(); - ctx.sql("CREATE TEMPORARY TABLE replaced AS SELECT * REPLACE (upper(value) AS value) FROM read_files('memory://files/data.txt')").await.unwrap(); - assert_eq!(columns(&ctx, "SELECT * FROM replaced").await, ["value"]); - let values = rows(&ctx, "SELECT value FROM replaced").await; - assert_eq!(values[0][0].try_as_str().flatten(), Some("HELLO")); + for (modifier, table) in [("TEMPORARY ", "temp_replaced"), ("", "replaced")] { + ctx.sql(&format!("CREATE {modifier}TABLE {table} AS SELECT * REPLACE (upper(value) AS value) FROM read_files('memory://files/data.txt')")).await.unwrap(); + assert_eq!( + columns(&ctx, &format!("SELECT * FROM {table}")).await, + ["value"] + ); + let values = rows(&ctx, &format!("SELECT value FROM {table}")).await; + assert_eq!(values[0][0].try_as_str().flatten(), Some("HELLO")); + } } #[tokio::test] diff --git a/crates/paimon/src/catalog/filesystem.rs b/crates/paimon/src/catalog/filesystem.rs index 1240f0c92..44d1af0a8 100644 --- a/crates/paimon/src/catalog/filesystem.rs +++ b/crates/paimon/src/catalog/filesystem.rs @@ -47,6 +47,15 @@ fn make_path(parent: &str, child: &str) -> String { format!("{parent}/{child}") } +fn serialize_table_schema(schema: &TableSchema) -> Result { + serde_json::to_vec(schema) + .map(Bytes::from) + .map_err(|error| Error::DataInvalid { + message: format!("Failed to serialize schema: {error}"), + source: Some(Box::new(error)), + }) +} + /// Filesystem catalog implementation. /// /// This catalog stores metadata on a filesystem with the following structure: @@ -259,15 +268,7 @@ impl FileSystemCatalog { self.file_io.mkdirs(&schema_dir).await?; let schema_path = self.schema_file_path(table_path, schema.id()); let output_file = self.file_io.new_output(&schema_path)?; - let content = - Bytes::from( - serde_json::to_string(schema).map_err(|e| Error::DataInvalid { - message: format!("Failed to serialize schema: {e}"), - source: Some(Box::new(e)), - })?, - ); - output_file.write(content).await?; - Ok(()) + output_file.write(serialize_table_schema(schema)?).await } /// Check if a database exists. @@ -463,11 +464,23 @@ impl Catalog for FileSystemCatalog { }); } - // todo: consider with lock if !table_exists { - self.file_io.mkdirs(&table_path).await?; + self.file_io + .mkdirs(&self.schema_dir_path(&table_path)) + .await?; let table_schema = TableSchema::new(0, &creation); - self.save_table_schema(&table_path, &table_schema).await?; + let created = self + .file_io + .write_atomic_if_not_exists( + &self.schema_file_path(&table_path, 0), + serialize_table_schema(&table_schema)?, + ) + .await?; + if !created && !ignore_if_exists { + return Err(Error::TableAlreadyExist { + full_name: identifier.full_name(), + }); + } } Ok(()) @@ -1752,6 +1765,80 @@ mod tests { ); } + #[tokio::test] + async fn test_create_table_atomically_rejects_a_raced_creator() { + #[derive(Debug)] + struct SharedMemoryStorage { + operator: opendal::Operator, + publication: tokio::sync::Barrier, + } + + #[async_trait] + impl crate::io::FileIOProvider for SharedMemoryStorage { + async fn create(&self, path: &str) -> Result<(opendal::Operator, String)> { + Ok(( + self.operator.clone(), + path.trim_start_matches("memory:/").into(), + )) + } + + async fn write_atomic_if_not_exists( + &self, + _path: &str, + _bytes: Bytes, + ) -> Result> { + self.publication.wait().await; + Ok(None) + } + } + + for ignore_if_exists in [false, true] { + let storage = std::sync::Arc::new(SharedMemoryStorage { + operator: + opendal::Operator::from_config(opendal::services::MemoryConfig::default()) + .unwrap(), + publication: tokio::sync::Barrier::new(2), + }); + let mut first = create_memory_catalog(); + let mut second = create_memory_catalog(); + first.file_io = first.file_io.clone().with_provider(storage.clone()); + second.file_io = second.file_io.clone().with_provider(storage); + first + .create_database("db", false, HashMap::new()) + .await + .unwrap(); + let identifier = Identifier::new("db", "raced"); + let schema = |name: &str| { + Schema::builder() + .column(name, DataType::Int(IntType::new())) + .build() + .unwrap() + }; + let (a, b) = tokio::time::timeout(std::time::Duration::from_secs(10), async { + tokio::join!( + first.create_table(&identifier, schema("first"), ignore_if_exists), + second.create_table(&identifier, schema("second"), ignore_if_exists), + ) + }) + .await + .unwrap(); + if ignore_if_exists { + a.unwrap(); + b.unwrap(); + } else { + assert_ne!(a.is_ok(), b.is_ok()); + let (winner, loser) = if a.is_ok() { + ("first", b) + } else { + ("second", a) + }; + assert!(matches!(loser, Err(Error::TableAlreadyExist { .. }))); + let table = first.get_table(&identifier).await.unwrap(); + assert_eq!(table.schema().fields()[0].name(), winner); + } + } + } + #[tokio::test] async fn test_create_table_rejects_an_unknown_type() { let (_temp_dir, catalog) = create_test_catalog(); diff --git a/crates/paimon/src/io/file_io.rs b/crates/paimon/src/io/file_io.rs index edc472a10..4a764dc33 100644 --- a/crates/paimon/src/io/file_io.rs +++ b/crates/paimon/src/io/file_io.rs @@ -997,7 +997,9 @@ impl FileIO { } else { return Err(Error::IoUnsupported { message: format!( - "Storage '{}' cannot atomically publish a complete snapshot without overwriting '{path}'; use a catalog SnapshotCommit with concurrency control", + "Storage '{}' cannot atomically publish '{path}' without overwriting it. \ + Use storage or a FileIOProvider with atomic publication support. \ + For snapshots, a catalog SnapshotCommit can also coordinate publication.", info.scheme() ), }); @@ -1028,14 +1030,14 @@ impl FileIO { ) -> Result { // OpenDAL's filesystem conditional write exposes the target before // its contents are complete. Stage locally, then publish without - // replacing another process's snapshot. + // replacing a file published by another process. let relative = normalize_path(relative_path); if std::path::Path::new(&relative) .components() .any(|component| matches!(component, std::path::Component::ParentDir)) { return Err(Error::IoUnsupported { - message: format!("Cannot publish a snapshot through a parent path: '{path}'"), + message: format!("Cannot atomically publish '{path}' through a parent path"), }); } let target = std::path::Path::new(&op.info().root()).join(relative); diff --git a/crates/paimon/src/table/data_file_writer.rs b/crates/paimon/src/table/data_file_writer.rs index a97ebea26..f920c4b7c 100644 --- a/crates/paimon/src/table/data_file_writer.rs +++ b/crates/paimon/src/table/data_file_writer.rs @@ -573,7 +573,9 @@ impl DataFileWriter { while self.in_flight_closes.join_next().await.is_some() {} for path in self.created_paths.drain(..) { if self.delete_file_upon_abort { - let _ = self.file_io.delete_file(&path).await; + if let Err(error) = self.file_io.delete_file(&path).await { + log::warn!("Failed to discard uncommitted file '{path}': {error}"); + } } } self.written_files.clear(); @@ -585,7 +587,9 @@ impl DataFileWriter { } for file in files { for path in file.collect_files(self.bucket_dir()) { - let _ = self.file_io.delete_file(&path).await; + if let Err(error) = self.file_io.delete_file(&path).await { + log::warn!("Failed to discard uncommitted file '{path}': {error}"); + } } } } diff --git a/crates/paimon/tests/mock_server.rs b/crates/paimon/tests/mock_server.rs index 6503331bf..715122578 100644 --- a/crates/paimon/tests/mock_server.rs +++ b/crates/paimon/tests/mock_server.rs @@ -220,7 +220,7 @@ fn paginate( #[derive(Clone)] pub struct RESTServer { warehouse: String, - _data_path: String, + data_path: String, config: ConfigResponse, inner: Arc>, resource_paths: ResourcePaths, @@ -233,7 +233,7 @@ impl RESTServer { /// Create a new RESTServer with initial databases. pub fn new( warehouse: String, - _data_path: String, + data_path: String, config: ConfigResponse, initial_dbs: Vec, ) -> Self { @@ -255,7 +255,7 @@ impl RESTServer { .collect(); RESTServer { - _data_path, + data_path, config, warehouse, inner: Arc::new(Mutex::new(MockState { @@ -828,15 +828,25 @@ impl RESTServer { return (StatusCode::CONFLICT, Json(err)).into_response(); } - // Create table response + let schema = match serde_json::from_value(payload["schema"].clone()) { + Ok(schema) => schema, + Err(error) => { + return ( + StatusCode::BAD_REQUEST, + Json(json!({"code": 400, "message": error.to_string()})), + ) + .into_response(); + } + }; + let path = format!("{}/{db}.db/{table_name}", state.data_path); let response = GetTableResponse::new( - Some(table_name.clone()), + Some(uuid::Uuid::new_v4().to_string()), Some(db.clone()), Some(table_name), - None, + Some(path), Some(true), - None, - None, + Some(0), + Some(schema), AuditRESTResponse::new(None, None, None, None, None), ); s.tables.insert(key, response); diff --git a/docs/src/sql.md b/docs/src/sql.md index 7996f2ae6..c29124b2c 100644 --- a/docs/src/sql.md +++ b/docs/src/sql.md @@ -43,7 +43,7 @@ SQL support has two layers: - DataFusion provides the parser, query planner, optimizer, execution engine, expressions, scalar functions, aggregate functions, and window functions. SQL statements that `SQLContext` does not intercept are delegated to DataFusion. This includes the DataFusion SQL surface for `SELECT` queries, CTEs (including recursive CTEs), subqueries, joins including `LATERAL` joins, SQL lambda functions, grouping, `HAVING`, window clauses, `QUALIFY`, set operations, `ORDER BY`, `LIMIT`/`OFFSET`, `EXPLAIN`, information-schema commands such as `SHOW TABLES`, `DESCRIBE`, `COPY`, and ordinary `INSERT`. - Paimon-specific table management and row-level writes are implemented by `SQLContext`. This includes Paimon `CREATE TABLE`, `ALTER TABLE`, `DROP TABLE`, `CREATE TEMPORARY TABLE`, `CREATE TEMPORARY VIEW`, REST Catalog persistent `CREATE VIEW`, `DROP VIEW`, `CREATE FUNCTION`, and `DROP FUNCTION`, `DROP TEMPORARY TABLE` / `VIEW`, `INSERT OVERWRITE ... PARTITION`, `UPDATE`, `DELETE`, `TRUNCATE TABLE`, `ALTER TABLE ... ADD PARTITION`, `ALTER TABLE ... DROP PARTITION`, `SHOW PARTITIONS`, `MSCK REPAIR TABLE`, `ANALYZE TABLE`, `CALL sys.*`, Paimon time travel, and `SET` / `RESET 'paimon.*'`. `MERGE INTO` uses DataFusion's native planner and the Paimon table provider for execution. -Not every DataFusion DDL/DML statement maps to a Paimon table operation. For Paimon catalogs, `CREATE EXTERNAL TABLE`, `LOCATION`, `CREATE MATERIALIZED VIEW`, and persistent `CREATE TABLE AS SELECT` are rejected or not implemented. Persistent `CREATE FUNCTION` is supported only for the REST Catalog SQL scalar form documented below. DataFusion `COPY` can export query results to files; it does not create or commit Paimon table files. +Not every DataFusion DDL/DML statement maps to a Paimon table operation. For Paimon catalogs, `CREATE EXTERNAL TABLE`, `LOCATION`, `CREATE MATERIALIZED VIEW`, and `CREATE OR REPLACE TABLE ... AS SELECT` are rejected or not implemented. Persistent `CREATE FUNCTION` is supported only for the REST Catalog SQL scalar form documented below. DataFusion `COPY` can export query results to files; it does not create or commit Paimon table files. For the exact delegated SQL grammar, see the [DataFusion SQL Reference](https://datafusion.apache.org/user-guide/sql/index.html). @@ -913,7 +913,62 @@ mixed-case or uppercase field when normalization is enabled. Unsupported syntax (will return an error): - `CREATE EXTERNAL TABLE` - `LOCATION` -- `CREATE TABLE AS SELECT` + +### CREATE TABLE AS SELECT + +Use a read-only query to create and populate a persistent Paimon table of the +default type (`'type' = 'table'`). + +```sql +CREATE TABLE paimon.my_db.query_result +PARTITIONED BY (region) +WITH ('bucket' = '-1') +AS SELECT id, region FROM source WHERE id > 10; +``` + +Column names, types and nullability come from the analyzed query, even when it +returns zero rows. Use aliases and casts to choose output names and types. +Geospatial outputs preserve their geometry or geography type, CRS and edge algorithm. +Table options, comments and partition keys use the same validation as +`CREATE TABLE`. Place `PARTITIONED BY` before `AS` and list unqualified output +column names. Quote a name containing a dot, as in `PARTITIONED BY ("event.day")`. +CTAS rejects explicit column declarations and `OR REPLACE`. The read-only +requirement also applies inside CTEs and set operations. Use one table-options +clause (`WITH`, `TBLPROPERTIES` or `OPTIONS`); multiple clauses and unsupported +modifiers such as `AUTO_INCREMENT` are rejected. + +Paimon stores `TIME` values with millisecond precision. CTAS accepts Arrow +second and millisecond time outputs, but rejects microsecond and nanosecond time +outputs before creating a table, including nested and dictionary-encoded outputs. +This also applies to empty results. DataFusion's `CAST(... AS TIME)` produces a +nanosecond time type. Cast it to `VARCHAR` to retain finer precision as text. + +Unqualified source tables and REST catalog SQL functions resolve in the +**current session catalog and database**, even for a destination in another +namespace. The same query plan supplies the schema and the rows. Source and +destination names follow the session's identifier normalization setting. +Quoted names retain their spelling. Use `VERSION AS OF` with a snapshot ID or +tag, or `TIMESTAMP AS OF` with a timestamp, to select a source version. +Tables in a join can read different snapshots. + +`IF NOT EXISTS` on an existing target skips source planning and execution, even +when that target uses an external table engine. Authorization and transport errors +are propagated. If another client creates the target first, CTAS returns a +table-already-exists error. With `IF NOT EXISTS` it succeeds without writing. +The filesystem catalog requires storage that can publish the initial schema +atomically without replacing an existing file, for both regular creation and +CTAS. After creation, CTAS loads the table and uses its normal write and commit +APIs. As with an ordinary INSERT, coordinate DROP, RENAME and table replacement +with active writers; CTAS does not isolate population from concurrent DDL. + +**A population failure retains the created table.** Other clients can write to +it as soon as creation succeeds. Before commit, a reported failure or caller +cancellation discards the writer's outstanding files. Cancellation allows an +in-progress write to finish before cleanup. Once commit starts, it finishes +even if the caller cancels, so the table may contain committed rows. If commit +returns an error, prepared files remain because a published snapshot may +reference them. Process termination can interrupt this cleanup. Inspect the +table and its snapshots before retrying. `IF NOT EXISTS` skips a retained table. ### Partitioned Tables