diff --git a/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/Spark42CompatibilityTest.scala b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/Spark42CompatibilityTest.scala index 3b3f6d511289..9fa402fd90a7 100644 --- a/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/Spark42CompatibilityTest.scala +++ b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/Spark42CompatibilityTest.scala @@ -104,6 +104,21 @@ class Spark42CompatibilityTest extends QueryTest with SharedSparkSession { } } + test("describe a binary partition") { + withTable("t") { + sql("CREATE TABLE t (id INT, p BINARY) PARTITIONED BY (p)") + sql("INSERT INTO t VALUES (1, CAST('2021' AS BINARY)), (2, CAST('2022' AS BINARY))") + // Spark renders a BINARY partition value as hex bytes by default. + withSQLConf("spark.sql.binaryOutputStyle" -> "UTF-8") { + checkAnswer( + sql("DESCRIBE FORMATTED t PARTITION (p = '2021')") + .filter("col_name = 'Partition Values'") + .select("data_type"), + Seq(Row("[p=2021]"))) + } + } + } + test("SQL scalar functions, builtin lookup and unsupported table function rejection") { try { sql("CREATE FUNCTION add_one(x INT) RETURNS INT RETURN x + 1") diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala index f53cefb93d5e..4183b2347c1d 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/PaimonPartitionManagement.scala @@ -18,9 +18,11 @@ package org.apache.paimon.spark -import org.apache.paimon.CoreOptions +import org.apache.paimon.{CoreOptions, Snapshot} +import org.apache.paimon.options.Options import org.apache.paimon.partition.PartitionStatistics import org.apache.paimon.table.{FileStoreTable, FormatTable, Table} +import org.apache.paimon.table.sink.CommitMessage import org.apache.paimon.table.source.ScanMode import org.apache.paimon.types.RowType import org.apache.paimon.utils.{InternalRowPartitionComputer, TypeUtils} @@ -31,8 +33,11 @@ import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow} import org.apache.spark.sql.catalyst.util.CharVarcharUtils import org.apache.spark.sql.connector.catalog.{SupportsAtomicPartitionManagement, TableCatalog} import org.apache.spark.sql.types.StructType +import org.apache.spark.unsafe.types.UTF8String -import java.util.{Map => JMap, Objects} +import java.nio.ByteBuffer +import java.nio.charset.StandardCharsets +import java.util.{ArrayList => JArrayList, Collections, List => JList, Map => JMap, Objects} import scala.collection.JavaConverters._ @@ -94,25 +99,162 @@ trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement with L new SparkRow(currentPartitionRowType, rowConverter(row).asInstanceOf[Row])) } + /** + * The first partition column whose value in `spec` does not parse back to its value in `row`. A + * spec holds each value as a string, and a blank one is written as the default partition name, + * which parses back as NULL. + */ + private def columnNotParsedBack(row: InternalRow, spec: JMap[String, String]): Option[String] = { + val names = table.partitionKeys().asScala.toSeq.take(row.numFields) + val specRowType = TypeUtils.project(table.rowType, names.asJava) + val parsed = InternalRowPartitionComputer.convertSpecToInternal( + spec, + specRowType, + CoreOptions.fromMap(table.options()).partitionDefaultName()) + names.zipWithIndex + .find { + case (name, index) => + val parsedValue = + DataConverter.fromPaimon(parsed.get(name), specRowType.getField(name).`type`()) + !Objects.deepEquals(row.get(index, partitionSchema(name).dataType), parsedValue) + } + .map(_._1) + } + + /** Whether the Spark value is a STRING or BINARY value equal to `text`. */ + private def isText(value: Any, text: String): Boolean = value match { + case s: UTF8String => s.toString == text + case b: Array[Byte] => new String(b, StandardCharsets.UTF_8) == text + case _ => false + } + + /** The first column of `row` whose value is the default partition name, which is not NULL. */ + private def columnHoldingDefaultPartitionName(row: InternalRow): Option[String] = { + val defaultPartitionName = CoreOptions.fromMap(table.options()).partitionDefaultName() + partitionSchema.fields.take(row.numFields).zipWithIndex.collectFirst { + case (field, i) if isText(row.get(i, field.dataType), defaultPartitionName) => field.name + } + } + + /** + * The first `numFields` values of a partition row, as a key that compares them the way + * `listPartitionIdentifiers` does: `equals` per value, and BINARY values by content. + */ + private def partitionKey(row: InternalRow, numFields: Int): JList[AnyRef] = { + val values = (0 until numFields).map { + i => + row.get(i, partitionSchema.fields(i).dataType) match { + case bytes: Array[Byte] => ByteBuffer.wrap(bytes) + case value => value + } + } + new JArrayList[AnyRef](values.asJava) + } + + /** The specs of the partitions to drop by name; each must address its own partition. */ + private def toRemovedPaimonPartitions(rows: Array[InternalRow]): Array[JMap[String, String]] = { + val specs = toPaimonPartitions(rows) + rows.zip(specs).foreach { + case (row, spec) => + columnNotParsedBack(row, spec).foreach { + name => + val nullPartitionHint = if (columnHoldingDefaultPartitionName(row).contains(name)) { + s" If you meant the NULL partition, use $name = NULL." + } else { + "" + } + throw new IllegalArgumentException( + s"Partition spec $spec of table ${table.name()} does not name the partition to " + + s"remove: the value of partition column $name does not parse back from it." + + nullPartitionHint) + } + } + specs + } + + /** + * Removes the data of the partitions in the latest snapshot that the rows address, matched by + * value: a full row addresses its own partition, a prefix row every partition under it. + */ + private def truncateStoredPartitions( + fileStoreTable: FileStoreTable, + rows: Array[InternalRow]): Unit = { + if (rows.isEmpty) { + return + } + // A batch scan can hide files, such as level 0 of a deletion-vector primary-key table; the + // snapshot reader lists every partition of the latest snapshot. + val stored = fileStoreTable + .newSnapshotReader() + .withMode(ScanMode.ALL) + .partitions() + .asScala + .map(partition => (partition, DataConverter.fromPaimon(partition, partitionRowType))) + val storedByPrefix = rows + .map(_.numFields) + .distinct + .map { + numFields => + numFields -> stored.groupBy { case (_, values) => partitionKey(values, numFields) } + } + .toMap + val partitions = rows.toSeq.flatMap { + row => + val matched = storedByPrefix(row.numFields) + .getOrElse(partitionKey(row, row.numFields), Seq.empty) + .map(_._1) + if (matched.isEmpty) { + columnHoldingDefaultPartitionName(row).foreach { + name => + throw new IllegalArgumentException( + s"No partition of table ${table.name()} has the default partition name as the " + + s"value of partition column $name. If you meant the NULL partition, use " + + s"$name = NULL.") + } + } + matched + }.distinct + if (partitions.nonEmpty) { + logInfo( + "Try to remove the data of partitions: " + partitions + .map(InternalRowPartitionComputer.partToSimpleString(partitionRowType, _, "/", 200)) + .mkString(",")) + // A static overwrite with no data removes the data of the given partitions. + val target = + fileStoreTable.copy(Map(CoreOptions.DYNAMIC_PARTITION_OVERWRITE.key -> "false").asJava) + val commit = target.newCommit(CoreOptions.createCommitUser(new Options(target.options()))) + try { + commit + .withOperation(Snapshot.Operation.TRUNCATE) + .withOverwriteStaticPartitions(partitions.asJava) + .commit(Collections.emptyList[CommitMessage]()) + } finally { + commit.close() + } + } + } + override def dropPartitions(rows: Array[InternalRow]): Boolean = { table match { case fileStoreTable: FileStoreTable => - val partitions = toPaimonPartitions(rows).toSeq.asJava - logInfo("Try to drop partitions: " + partitions.asScala.mkString(",")) - val partitionModification = fileStoreTable.catalogEnvironment().partitionModification() + // The catalog drops the partitions it keeps in a metastore by name. + val options = fileStoreTable.coreOptions() + val partitionModification = + if (options.partitionedTableInMetastore() || options.tagToPartitionField() != null) { + fileStoreTable.catalogEnvironment().partitionModification() + } else { + null + } if (partitionModification != null) { try { + val partitions = toRemovedPaimonPartitions(rows).toSeq.asJava + logInfo("Try to drop partitions: " + partitions.asScala.mkString(",")) partitionModification.dropPartitions(partitions) } finally { partitionModification.close() } } else { - val commit = fileStoreTable.newBatchWriteBuilder().newCommit() - try { - commit.truncatePartitions(partitions) - } finally { - commit.close() - } + truncateStoredPartitions(fileStoreTable, rows) } true @@ -122,12 +264,17 @@ trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement with L } override def truncatePartitions(idents: Array[InternalRow]): Boolean = { - val partitions = toPaimonPartitions(idents).toSeq.asJava - val commit = table.newBatchWriteBuilder().newCommit() - try { - commit.truncatePartitions(partitions) - } finally { - commit.close() + table match { + case fileStoreTable: FileStoreTable => + truncateStoredPartitions(fileStoreTable, idents) + case _ => + val partitions = toPaimonPartitions(idents).toSeq.asJava + val commit = table.newBatchWriteBuilder().newCommit() + try { + commit.truncatePartitions(partitions) + } finally { + commit.close() + } } true } @@ -146,24 +293,30 @@ trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement with L table match { case fileStoreTable: FileStoreTable => val partitionSpec = toPaimonPartitions(Array(ident)).head - val partitionEntries = fileStoreTable - .newSnapshotReader() - .withMode(ScanMode.ALL) - .withPartitionFilter(partitionSpec) - .partitionEntries() - - if (!partitionEntries.isEmpty) { - val entry = partitionEntries.get(0) - Map( - PartitionStatistics.FIELD_RECORD_COUNT -> entry.recordCount().toString, - PartitionStatistics.FIELD_FILE_SIZE_IN_BYTES -> entry.fileSizeInBytes().toString, - PartitionStatistics.FIELD_FILE_COUNT -> entry.fileCount().toString, - PartitionStatistics.FIELD_LAST_FILE_CREATION_TIME -> entry - .lastFileCreationTime() - .toString - ).asJava + val reader = fileStoreTable.newSnapshotReader().withMode(ScanMode.ALL) + val partitionEntry = if (columnNotParsedBack(ident, partitionSpec).isEmpty) { + reader.withPartitionFilter(partitionSpec).partitionEntries().asScala.headOption } else { - Map.empty[String, String].asJava + // The spec may select another partition or none, so match the partitions by value. + reader.partitionEntries().asScala.find { + entry => + val partition = DataConverter.fromPaimon(entry.partition(), partitionRowType) + partitionKey(partition, ident.numFields) == partitionKey(ident, ident.numFields) + } + } + + partitionEntry match { + case Some(entry) => + Map( + PartitionStatistics.FIELD_RECORD_COUNT -> entry.recordCount().toString, + PartitionStatistics.FIELD_FILE_SIZE_IN_BYTES -> entry.fileSizeInBytes().toString, + PartitionStatistics.FIELD_FILE_COUNT -> entry.fileCount().toString, + PartitionStatistics.FIELD_LAST_FILE_CREATION_TIME -> entry + .lastFileCreationTime() + .toString + ).asJava + case None => + Map.empty[String, String].asJava } case formatTable: FormatTable => val partitionManager = formatTable.partitionManager() @@ -233,7 +386,8 @@ trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement with L case (partitionName, index) => val internalRowIndex = partitionSchema.fieldIndex(partitionName) val structField = partitionSchema.fields(internalRowIndex) - Objects.equals( + // `deepEquals`: a BINARY partition value is a `byte[]`. + Objects.deepEquals( sparkInternalRow.get(internalRowIndex, structField.dataType), internalRow.get(index, structField.dataType)) } diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala index feae8be0dab5..f09741fd99b3 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala @@ -138,6 +138,40 @@ abstract class DDLWithHiveCatalogTestBase extends PaimonHiveTestBase { } } + test("Paimon DDL with hive catalog: drop a blank partition") { + spark.sql(s"USE $paimonHiveCatalogName") + withDatabase("paimon_db") { + spark.sql("CREATE DATABASE paimon_db") + Seq("false", "true").foreach { + inMetastore => + withTable("paimon_db.blank_tbl") { + spark.sql( + "CREATE TABLE paimon_db.blank_tbl (a INT, p STRING) USING PAIMON " + + "PARTITIONED BY (p) " + + s"TBLPROPERTIES ('metastore.partitioned-table' = '$inMetastore')") + // In a metastore, blank and NULL have the same partition name, so one commit cannot + // add both. + Seq("(1, ' ')", "(2, NULL)", "(3, 'a')").foreach { + row => spark.sql(s"INSERT INTO paimon_db.blank_tbl VALUES $row") + } + val drop = "ALTER TABLE paimon_db.blank_tbl DROP PARTITION (p = ' ')" + if (inMetastore.toBoolean) { + // The metastore names a blank partition as it names the null one. + val e = intercept[IllegalArgumentException](spark.sql(drop)) + assert(e.getMessage.contains("does not parse back")) + assert(!e.getMessage.contains("= NULL")) + checkAnswer( + spark.sql("SELECT a FROM paimon_db.blank_tbl"), + Row(1) :: Row(2) :: Row(3) :: Nil) + } else { + spark.sql(drop) + checkAnswer(spark.sql("SELECT a FROM paimon_db.blank_tbl"), Row(2) :: Row(3) :: Nil) + } + } + } + } + } + test( "Paimon partition expire test with hive catalog: expire partition for paimon table sparkCatalogName") { spark.sql(s"USE $paimonHiveCatalogName") diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonPartitionManagementTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonPartitionManagementTest.scala index eebb3a801393..778be8401ac8 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonPartitionManagementTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonPartitionManagementTest.scala @@ -22,6 +22,8 @@ import org.apache.paimon.spark.PaimonSparkTestBase import org.apache.spark.sql.{AnalysisException, Row} +import scala.collection.JavaConverters._ + class PaimonPartitionManagementTest extends PaimonSparkTestBase { withPk.foreach { @@ -211,6 +213,133 @@ class PaimonPartitionManagementTest extends PaimonSparkTestBase { ) } + test("Paimon Partition Management: show, add and drop a binary partition") { + withTable("T") { + spark.sql("CREATE TABLE T (a INT, p BINARY) PARTITIONED BY (p)") + spark.sql("INSERT INTO T VALUES (1, CAST('2021' AS BINARY)), (2, CAST('2022' AS BINARY))") + checkAnswer(spark.sql("SHOW PARTITIONS T PARTITION (p = '2021')"), Row("p=2021") :: Nil) + + val e = intercept[AnalysisException] { + spark.sql("ALTER TABLE T ADD PARTITION (p = '2021')") + } + assert(e.getMessage.contains("already exist")) + + spark.sql("ALTER TABLE T DROP PARTITION (p = '2021')") + checkAnswer(spark.sql("SHOW PARTITIONS T"), Row("p=2022") :: Nil) + } + } + + test("Paimon Partition Management: truncate with a partial binary partition spec") { + withTable("T") { + spark.sql("CREATE TABLE T (a INT, p1 BINARY, p2 STRING) PARTITIONED BY (p1, p2)") + spark.sql( + "INSERT INTO T VALUES (1, CAST('x' AS BINARY), 'a'), (2, CAST('x' AS BINARY), 'b'), " + + "(3, CAST('y' AS BINARY), 'a')") + spark.sql("TRUNCATE TABLE T PARTITION (p1 = 'x')") + checkAnswer(spark.sql("SELECT a FROM T"), Row(3) :: Nil) + // A partial spec that matches no partition truncates nothing. + spark.sql("TRUNCATE TABLE T PARTITION (p1 = 'z')") + checkAnswer(spark.sql("SELECT a FROM T"), Row(3) :: Nil) + } + } + + Seq("BINARY", "STRING").foreach { + partitionType => + test(s"Paimon Partition Management: drop a blank $partitionType partition") { + withTable("T") { + spark.sql(s"CREATE TABLE T (a INT, p $partitionType) PARTITIONED BY (p)") + spark.sql( + s"INSERT INTO T VALUES (1, CAST(' ' AS $partitionType)), (2, NULL), " + + s"(3, CAST('a' AS $partitionType))") + // A blank value's partition name is the null partition's. + spark.sql("ALTER TABLE T DROP PARTITION (p = ' ')") + checkAnswer(spark.sql("SELECT a FROM T"), Row(2) :: Row(3) :: Nil) + } + } + } + + test("Paimon Partition Management: truncate binary partitions whose names do not parse back") { + withTable("T") { + spark.sql("CREATE TABLE T (a INT, p1 BINARY, p2 STRING) PARTITIONED BY (p1, p2)") + spark.sql( + "INSERT INTO T VALUES (1, CAST(' ' AS BINARY), 'a'), (2, NULL, 'a'), " + + "(3, X'FF', 'b'), (4, CAST('x' AS BINARY), 'b')") + spark.sql("TRUNCATE TABLE T PARTITION (p1 = ' ')") + checkAnswer(spark.sql("SELECT a FROM T"), Row(2) :: Row(3) :: Row(4) :: Nil) + // X'FF' is not valid UTF-8, so its partition name holds other bytes. + spark.sql("TRUNCATE TABLE T PARTITION (p2 = 'b')") + checkAnswer(spark.sql("SELECT a FROM T"), Row(2) :: Nil) + } + } + + test("Paimon Partition Management: truncate a partition that a batch read does not see") { + withTable("T") { + spark.sql( + "CREATE TABLE T (k INT, p STRING) PARTITIONED BY (p) TBLPROPERTIES ('primary-key' = " + + "'k,p', 'bucket' = '1', 'deletion-vectors.enabled' = 'true', 'write-only' = 'true')") + spark.sql("INSERT INTO T VALUES (1, 'a'), (2, 'b')") + // A batch read skips level 0 of a deletion-vector table, which holds every file here. + assert(loadTable("T").newReadBuilder().newScan().listPartitions().isEmpty) + def partitions: Seq[String] = + loadTable("T").newSnapshotReader().partitions().asScala.map(_.getString(0).toString).toSeq + assert(partitions.sorted == Seq("a", "b")) + spark.sql("TRUNCATE TABLE T PARTITION (p = 'a')") + assert(partitions == Seq("b")) + } + } + + test("Paimon Partition Management: drop a timestamp partition whose legacy name is shifted") { + withTimeZone("America/Los_Angeles") { + withTable("T") { + spark.sql("CREATE TABLE T (a INT, p TIMESTAMP) PARTITIONED BY (p)") + // The legacy name of 10:30:01 is its UTC wall clock, which parses back as 17:30:01 here. + spark.sql( + "INSERT INTO T VALUES (1, CAST('2024-06-01 10:30:01' AS TIMESTAMP)), " + + "(2, CAST('2024-06-01 17:30:01' AS TIMESTAMP))") + spark.sql("ALTER TABLE T DROP PARTITION (p = '2024-06-01 10:30:01')") + checkAnswer(spark.sql("SELECT a FROM T"), Row(2) :: Nil) + } + } + } + + test("Paimon Partition Management: address the null partition by NULL, not its default name") { + withTable("T") { + spark.sql("CREATE TABLE T (a INT, p1 STRING, p2 STRING) PARTITIONED BY (p1, p2)") + spark.sql("INSERT INTO T VALUES (1, NULL, 'x'), (2, 'b', 'x')") + val truncate = intercept[IllegalArgumentException] { + spark.sql("TRUNCATE TABLE T PARTITION (p1 = '__DEFAULT_PARTITION__', p2 = 'x')") + } + assert(truncate.getMessage.contains("use p1 = NULL")) + val drop = intercept[IllegalArgumentException] { + spark.sql("ALTER TABLE T DROP PARTITION (p1 = '__DEFAULT_PARTITION__')") + } + assert(drop.getMessage.contains("use p1 = NULL")) + checkAnswer(spark.sql("SELECT a FROM T"), Row(1) :: Row(2) :: Nil) + + spark.sql("ALTER TABLE T DROP PARTITION (p1 = NULL)") + checkAnswer(spark.sql("SELECT a FROM T"), Row(2) :: Nil) + } + } + + Seq("BINARY", "STRING").foreach { + partitionType => + test(s"Paimon Partition Management: show the statistics of a blank $partitionType partition") { + withTable("T") { + spark.sql(s"CREATE TABLE T (a INT, p $partitionType) PARTITIONED BY (p)") + spark.sql(s"INSERT INTO T VALUES (1, CAST(' ' AS $partitionType)), (2, NULL), (3, NULL)") + def information(spec: String): String = + spark + .sql(s"SHOW TABLE EXTENDED IN $dbName0 LIKE 'T' PARTITION ($spec)") + .select("information") + .collect() + .head + .getString(0) + assert(information("p = ' '").contains("Partition Statistics: 1 rows")) + assert(information("p = NULL").contains("Partition Statistics: 2 rows")) + } + } + } + test("Paimon Partition Management: timestamp partition column") { spark.sql(s""" |CREATE TABLE T (a INT, dt TIMESTAMP)