From 50b51ba319b3467c4c4d25372b94511dbd8eed5a Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Thu, 19 Nov 2026 13:44:06 +0800 Subject: [PATCH] [spark] Reject table constraints in column-list REPLACE TABLE on Spark 4.1+ --- .../spark/sql/Spark42CompatibilityTest.scala | 16 +++++++++++++++- .../PaimonReplaceTableAsSelectStrategy.scala | 1 + .../spark/sql/paimon/shims/SparkShim.scala | 6 ++++++ .../apache/paimon/spark/sql/DDLTestBase.scala | 13 +++++++++++++ .../spark/sql/paimon/shims/Spark4Shim.scala | 9 ++++++++- 5 files changed, 43 insertions(+), 2 deletions(-) diff --git a/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/Spark42CompatibilityTest.scala b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/Spark42CompatibilityTest.scala index da37db0434d3..d535c2f6c275 100644 --- a/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/Spark42CompatibilityTest.scala +++ b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/Spark42CompatibilityTest.scala @@ -25,7 +25,7 @@ import org.apache.paimon.spark.SparkCatalog import org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions import org.apache.spark.SparkConf -import org.apache.spark.sql.{QueryTest, Row} +import org.apache.spark.sql.{AnalysisException, QueryTest, Row} import org.apache.spark.sql.test.SharedSparkSession import java.nio.file.Files @@ -86,6 +86,20 @@ class Spark42CompatibilityTest extends QueryTest with SharedSparkSession { } } + test("replace table rejects table constraints and keeps the old table") { + withTable("t") { + sql("CREATE TABLE t (id INT) TBLPROPERTIES ('file.format'='avro')") + sql("INSERT INTO t VALUES (7)") + val e = intercept[AnalysisException] { + sql( + "CREATE OR REPLACE TABLE t (id INT, CONSTRAINT positive CHECK (id > 0)) " + + "TBLPROPERTIES ('file.format'='avro')") + } + assert(e.getCondition === "UNSUPPORTED_FEATURE.TABLE_OPERATION") + checkAnswer(sql("SELECT * FROM t"), Seq(Row(7))) + } + } + test("ordinary views, table description and partition description") { withTable("t") { sql( diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/execution/shim/PaimonReplaceTableAsSelectStrategy.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/execution/shim/PaimonReplaceTableAsSelectStrategy.scala index 5c4b091d2a05..314b596412dd 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/execution/shim/PaimonReplaceTableAsSelectStrategy.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/execution/shim/PaimonReplaceTableAsSelectStrategy.scala @@ -109,6 +109,7 @@ case class PaimonReplaceTableStrategy(spark: SparkSession) parts, tableSpec: TableSpec, orCreate) if supportsCatalog(catalog, tableSpec.provider.orNull) => + SparkShimLoader.shim.validateTableConstraints(tableSpec, catalog, ident) val columns = SparkShimLoader.shim.toReplaceTableColumns( replace.tableSchema, diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/paimon/shims/SparkShim.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/paimon/shims/SparkShim.scala index a3095c1b50f6..e40706affa9c 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/paimon/shims/SparkShim.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/paimon/shims/SparkShim.scala @@ -127,6 +127,12 @@ trait SparkShim { catalog: TableCatalog, ident: Identifier): Array[Column] + // Spark 4.1 added table constraints. Earlier versions have none to validate. + def validateTableConstraints( + tableSpec: TableSpec, + catalog: TableCatalog, + ident: Identifier): Unit = {} + def copyTableSpec( tableSpec: TableSpec, additionalProperties: Map[String, String], diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala index b8cfd81d9d65..12603eb266d3 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLTestBase.scala @@ -344,6 +344,19 @@ abstract class DDLTestBase extends PaimonSparkTestBase { } } + test("Paimon DDL: REPLACE TABLE rejects table constraints and keeps the old table") { + assume(gteqSpark4_1) + withTable("t") { + sql("CREATE TABLE t (id INT) USING paimon") + sql("INSERT INTO t VALUES (7)") + val error = intercept[AnalysisException] { + sql("CREATE OR REPLACE TABLE t (id INT, CONSTRAINT positive CHECK (id > 0)) USING paimon") + } + Assertions.assertEquals("UNSUPPORTED_FEATURE.TABLE_OPERATION", error.getErrorClass) + checkAnswer(sql("SELECT * FROM t"), Seq(Row(7))) + } + } + test("Paimon DDL: CREATE TABLE fails when table exists") { withTable("t") { sql("CREATE TABLE t (id BIGINT, data STRING) USING paimon") diff --git a/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala index 6b55d4673996..9731b2d22d82 100644 --- a/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala +++ b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala @@ -42,7 +42,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Assignment, Colum import org.apache.spark.sql.catalyst.plans.logical.MergeRows.{Copy, Insert, Keep, Update} import org.apache.spark.sql.catalyst.plans.physical.{ClusteredDistribution, Distribution} import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.catalyst.util.{ArrayData, GeneratedColumn, IdentityColumn, ResolveDefaultColumns, STUtils} +import org.apache.spark.sql.catalyst.util.{ArrayData, GeneratedColumn, IdentityColumn, ResolveDefaultColumns, ResolveTableConstraints, STUtils} import org.apache.spark.sql.connector.catalog.{CatalogPlugin, CatalogV2Util, Column, Identifier, StagingTableCatalog, Table, TableCatalog} import org.apache.spark.sql.connector.expressions.{Expression => V2Expression, Transform} import org.apache.spark.sql.connector.read.Scan @@ -201,6 +201,13 @@ class Spark4Shim extends SparkShim { columns.map(_.toV2Column(statementType)).toArray } + override def validateTableConstraints( + tableSpec: TableSpec, + catalog: TableCatalog, + ident: Identifier): Unit = { + ResolveTableConstraints.validateCatalogForTableConstraint(tableSpec.constraints, catalog, ident) + } + override def copyTableSpec( tableSpec: TableSpec, additionalProperties: Map[String, String],