Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import org.apache.paimon.spark.SparkCatalog
import org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions

import org.apache.spark.SparkConf
import org.apache.spark.sql.{QueryTest, Row}
import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
import org.apache.spark.sql.test.SharedSparkSession

import java.nio.file.Files
Expand Down Expand Up @@ -86,6 +86,20 @@ class Spark42CompatibilityTest extends QueryTest with SharedSparkSession {
}
}

test("replace table rejects table constraints and keeps the old table") {
withTable("t") {
sql("CREATE TABLE t (id INT) TBLPROPERTIES ('file.format'='avro')")
sql("INSERT INTO t VALUES (7)")
val e = intercept[AnalysisException] {
sql(
"CREATE OR REPLACE TABLE t (id INT, CONSTRAINT positive CHECK (id > 0)) " +
"TBLPROPERTIES ('file.format'='avro')")
}
assert(e.getCondition === "UNSUPPORTED_FEATURE.TABLE_OPERATION")
checkAnswer(sql("SELECT * FROM t"), Seq(Row(7)))
}
}

test("ordinary views, table description and partition description") {
withTable("t") {
sql(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ case class PaimonReplaceTableStrategy(spark: SparkSession)
parts,
tableSpec: TableSpec,
orCreate) if supportsCatalog(catalog, tableSpec.provider.orNull) =>
SparkShimLoader.shim.validateTableConstraints(tableSpec, catalog, ident)
val columns =
SparkShimLoader.shim.toReplaceTableColumns(
replace.tableSchema,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,12 @@ trait SparkShim {
catalog: TableCatalog,
ident: Identifier): Array[Column]

// Spark 4.1 added table constraints. Earlier versions have none to validate.
def validateTableConstraints(
tableSpec: TableSpec,
catalog: TableCatalog,
ident: Identifier): Unit = {}

def copyTableSpec(
tableSpec: TableSpec,
additionalProperties: Map[String, String],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,19 @@ abstract class DDLTestBase extends PaimonSparkTestBase {
}
}

test("Paimon DDL: REPLACE TABLE rejects table constraints and keeps the old table") {
assume(gteqSpark4_1)
withTable("t") {
sql("CREATE TABLE t (id INT) USING paimon")
sql("INSERT INTO t VALUES (7)")
val error = intercept[AnalysisException] {
sql("CREATE OR REPLACE TABLE t (id INT, CONSTRAINT positive CHECK (id > 0)) USING paimon")
}
Assertions.assertEquals("UNSUPPORTED_FEATURE.TABLE_OPERATION", error.getErrorClass)
checkAnswer(sql("SELECT * FROM t"), Seq(Row(7)))
}
}

test("Paimon DDL: CREATE TABLE fails when table exists") {
withTable("t") {
sql("CREATE TABLE t (id BIGINT, data STRING) USING paimon")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Assignment, Colum
import org.apache.spark.sql.catalyst.plans.logical.MergeRows.{Copy, Insert, Keep, Update}
import org.apache.spark.sql.catalyst.plans.physical.{ClusteredDistribution, Distribution}
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.catalyst.util.{ArrayData, GeneratedColumn, IdentityColumn, ResolveDefaultColumns, STUtils}
import org.apache.spark.sql.catalyst.util.{ArrayData, GeneratedColumn, IdentityColumn, ResolveDefaultColumns, ResolveTableConstraints, STUtils}
import org.apache.spark.sql.connector.catalog.{CatalogPlugin, CatalogV2Util, Column, Identifier, StagingTableCatalog, Table, TableCatalog}
import org.apache.spark.sql.connector.expressions.{Expression => V2Expression, Transform}
import org.apache.spark.sql.connector.read.Scan
Expand Down Expand Up @@ -201,6 +201,13 @@ class Spark4Shim extends SparkShim {
columns.map(_.toV2Column(statementType)).toArray
}

override def validateTableConstraints(
tableSpec: TableSpec,
catalog: TableCatalog,
ident: Identifier): Unit = {
ResolveTableConstraints.validateCatalogForTableConstraint(tableSpec.constraints, catalog, ident)
}

override def copyTableSpec(
tableSpec: TableSpec,
additionalProperties: Map[String, String],
Expand Down
Loading