Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,21 @@ class Spark42CompatibilityTest extends QueryTest with SharedSparkSession {
}
}

test("describe a binary partition") {
withTable("t") {
sql("CREATE TABLE t (id INT, p BINARY) PARTITIONED BY (p)")
sql("INSERT INTO t VALUES (1, CAST('2021' AS BINARY)), (2, CAST('2022' AS BINARY))")
// Spark renders a BINARY partition value as hex bytes by default.
withSQLConf("spark.sql.binaryOutputStyle" -> "UTF-8") {
checkAnswer(
sql("DESCRIBE FORMATTED t PARTITION (p = '2021')")
.filter("col_name = 'Partition Values'")
.select("data_type"),
Seq(Row("[p=2021]")))
}
}
}

test("SQL scalar functions, builtin lookup and unsupported table function rejection") {
try {
sql("CREATE FUNCTION add_one(x INT) RETURNS INT RETURN x + 1")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@

package org.apache.paimon.spark

import org.apache.paimon.CoreOptions
import org.apache.paimon.{CoreOptions, Snapshot}
import org.apache.paimon.options.Options
import org.apache.paimon.partition.PartitionStatistics
import org.apache.paimon.table.{FileStoreTable, FormatTable, Table}
import org.apache.paimon.table.sink.CommitMessage
import org.apache.paimon.table.source.ScanMode
import org.apache.paimon.types.RowType
import org.apache.paimon.utils.{InternalRowPartitionComputer, TypeUtils}
Expand All @@ -31,8 +33,11 @@ import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow}
import org.apache.spark.sql.catalyst.util.CharVarcharUtils
import org.apache.spark.sql.connector.catalog.{SupportsAtomicPartitionManagement, TableCatalog}
import org.apache.spark.sql.types.StructType
import org.apache.spark.unsafe.types.UTF8String

import java.util.{Map => JMap, Objects}
import java.nio.ByteBuffer
import java.nio.charset.StandardCharsets
import java.util.{ArrayList => JArrayList, Collections, List => JList, Map => JMap, Objects}

import scala.collection.JavaConverters._

Expand Down Expand Up @@ -94,25 +99,162 @@ trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement with L
new SparkRow(currentPartitionRowType, rowConverter(row).asInstanceOf[Row]))
}

/**
* The first partition column whose value in `spec` does not parse back to its value in `row`. A
* spec holds each value as a string, and a blank one is written as the default partition name,
* which parses back as NULL.
*/
private def columnNotParsedBack(row: InternalRow, spec: JMap[String, String]): Option[String] = {
val names = table.partitionKeys().asScala.toSeq.take(row.numFields)
val specRowType = TypeUtils.project(table.rowType, names.asJava)
val parsed = InternalRowPartitionComputer.convertSpecToInternal(
spec,
specRowType,
CoreOptions.fromMap(table.options()).partitionDefaultName())
names.zipWithIndex
.find {
case (name, index) =>
val parsedValue =
DataConverter.fromPaimon(parsed.get(name), specRowType.getField(name).`type`())
!Objects.deepEquals(row.get(index, partitionSchema(name).dataType), parsedValue)
}
.map(_._1)
}

/** Whether the Spark value is a STRING or BINARY value equal to `text`. */
private def isText(value: Any, text: String): Boolean = value match {
case s: UTF8String => s.toString == text
case b: Array[Byte] => new String(b, StandardCharsets.UTF_8) == text
case _ => false
}

/** The first column of `row` whose value is the default partition name, which is not NULL. */
private def columnHoldingDefaultPartitionName(row: InternalRow): Option[String] = {
val defaultPartitionName = CoreOptions.fromMap(table.options()).partitionDefaultName()
partitionSchema.fields.take(row.numFields).zipWithIndex.collectFirst {
case (field, i) if isText(row.get(i, field.dataType), defaultPartitionName) => field.name
}
}

/**
* The first `numFields` values of a partition row, as a key that compares them the way
* `listPartitionIdentifiers` does: `equals` per value, and BINARY values by content.
*/
private def partitionKey(row: InternalRow, numFields: Int): JList[AnyRef] = {
val values = (0 until numFields).map {
i =>
row.get(i, partitionSchema.fields(i).dataType) match {
case bytes: Array[Byte] => ByteBuffer.wrap(bytes)
case value => value
}
}
new JArrayList[AnyRef](values.asJava)
}

/** The specs of the partitions to drop by name; each must address its own partition. */
private def toRemovedPaimonPartitions(rows: Array[InternalRow]): Array[JMap[String, String]] = {
val specs = toPaimonPartitions(rows)
rows.zip(specs).foreach {
case (row, spec) =>
columnNotParsedBack(row, spec).foreach {
name =>
val nullPartitionHint = if (columnHoldingDefaultPartitionName(row).contains(name)) {
s" If you meant the NULL partition, use $name = NULL."
} else {
""
}
throw new IllegalArgumentException(
s"Partition spec $spec of table ${table.name()} does not name the partition to " +
s"remove: the value of partition column $name does not parse back from it." +
nullPartitionHint)
}
}
specs
}

/**
* Removes the data of the partitions in the latest snapshot that the rows address, matched by
* value: a full row addresses its own partition, a prefix row every partition under it.
*/
private def truncateStoredPartitions(
fileStoreTable: FileStoreTable,
rows: Array[InternalRow]): Unit = {
if (rows.isEmpty) {
return
}
// A batch scan can hide files, such as level 0 of a deletion-vector primary-key table; the
// snapshot reader lists every partition of the latest snapshot.
val stored = fileStoreTable
.newSnapshotReader()
.withMode(ScanMode.ALL)
.partitions()
.asScala
.map(partition => (partition, DataConverter.fromPaimon(partition, partitionRowType)))
val storedByPrefix = rows
.map(_.numFields)
.distinct
.map {
numFields =>
numFields -> stored.groupBy { case (_, values) => partitionKey(values, numFields) }
}
.toMap
val partitions = rows.toSeq.flatMap {
row =>
val matched = storedByPrefix(row.numFields)
.getOrElse(partitionKey(row, row.numFields), Seq.empty)
.map(_._1)
if (matched.isEmpty) {
columnHoldingDefaultPartitionName(row).foreach {
name =>
throw new IllegalArgumentException(
s"No partition of table ${table.name()} has the default partition name as the " +
s"value of partition column $name. If you meant the NULL partition, use " +
s"$name = NULL.")
}
}
matched
}.distinct
if (partitions.nonEmpty) {
logInfo(
"Try to remove the data of partitions: " + partitions
.map(InternalRowPartitionComputer.partToSimpleString(partitionRowType, _, "/", 200))
.mkString(","))
// A static overwrite with no data removes the data of the given partitions.
val target =
fileStoreTable.copy(Map(CoreOptions.DYNAMIC_PARTITION_OVERWRITE.key -> "false").asJava)
val commit = target.newCommit(CoreOptions.createCommitUser(new Options(target.options())))
try {
commit
.withOperation(Snapshot.Operation.TRUNCATE)
.withOverwriteStaticPartitions(partitions.asJava)
.commit(Collections.emptyList[CommitMessage]())
} finally {
commit.close()
}
}
}

override def dropPartitions(rows: Array[InternalRow]): Boolean = {
table match {
case fileStoreTable: FileStoreTable =>
val partitions = toPaimonPartitions(rows).toSeq.asJava
logInfo("Try to drop partitions: " + partitions.asScala.mkString(","))
val partitionModification = fileStoreTable.catalogEnvironment().partitionModification()
// The catalog drops the partitions it keeps in a metastore by name.
val options = fileStoreTable.coreOptions()
val partitionModification =
if (options.partitionedTableInMetastore() || options.tagToPartitionField() != null) {
fileStoreTable.catalogEnvironment().partitionModification()
} else {
null
}
if (partitionModification != null) {
try {
val partitions = toRemovedPaimonPartitions(rows).toSeq.asJava
logInfo("Try to drop partitions: " + partitions.asScala.mkString(","))
partitionModification.dropPartitions(partitions)
} finally {
partitionModification.close()
}
} else {
val commit = fileStoreTable.newBatchWriteBuilder().newCommit()
try {
commit.truncatePartitions(partitions)
} finally {
commit.close()
}
truncateStoredPartitions(fileStoreTable, rows)
}
true

Expand All @@ -122,12 +264,17 @@ trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement with L
}

override def truncatePartitions(idents: Array[InternalRow]): Boolean = {
val partitions = toPaimonPartitions(idents).toSeq.asJava
val commit = table.newBatchWriteBuilder().newCommit()
try {
commit.truncatePartitions(partitions)
} finally {
commit.close()
table match {
case fileStoreTable: FileStoreTable =>
truncateStoredPartitions(fileStoreTable, idents)
case _ =>
val partitions = toPaimonPartitions(idents).toSeq.asJava
val commit = table.newBatchWriteBuilder().newCommit()
try {
commit.truncatePartitions(partitions)
} finally {
commit.close()
}
}
true
}
Expand All @@ -146,24 +293,30 @@ trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement with L
table match {
case fileStoreTable: FileStoreTable =>
val partitionSpec = toPaimonPartitions(Array(ident)).head
val partitionEntries = fileStoreTable
.newSnapshotReader()
.withMode(ScanMode.ALL)
.withPartitionFilter(partitionSpec)
.partitionEntries()

if (!partitionEntries.isEmpty) {
val entry = partitionEntries.get(0)
Map(
PartitionStatistics.FIELD_RECORD_COUNT -> entry.recordCount().toString,
PartitionStatistics.FIELD_FILE_SIZE_IN_BYTES -> entry.fileSizeInBytes().toString,
PartitionStatistics.FIELD_FILE_COUNT -> entry.fileCount().toString,
PartitionStatistics.FIELD_LAST_FILE_CREATION_TIME -> entry
.lastFileCreationTime()
.toString
).asJava
val reader = fileStoreTable.newSnapshotReader().withMode(ScanMode.ALL)
val partitionEntry = if (columnNotParsedBack(ident, partitionSpec).isEmpty) {
reader.withPartitionFilter(partitionSpec).partitionEntries().asScala.headOption
} else {
Map.empty[String, String].asJava
// The spec may select another partition or none, so match the partitions by value.
reader.partitionEntries().asScala.find {
entry =>
val partition = DataConverter.fromPaimon(entry.partition(), partitionRowType)
partitionKey(partition, ident.numFields) == partitionKey(ident, ident.numFields)
}
}

partitionEntry match {
case Some(entry) =>
Map(
PartitionStatistics.FIELD_RECORD_COUNT -> entry.recordCount().toString,
PartitionStatistics.FIELD_FILE_SIZE_IN_BYTES -> entry.fileSizeInBytes().toString,
PartitionStatistics.FIELD_FILE_COUNT -> entry.fileCount().toString,
PartitionStatistics.FIELD_LAST_FILE_CREATION_TIME -> entry
.lastFileCreationTime()
.toString
).asJava
case None =>
Map.empty[String, String].asJava
}
case formatTable: FormatTable =>
val partitionManager = formatTable.partitionManager()
Expand Down Expand Up @@ -233,7 +386,8 @@ trait PaimonPartitionManagement extends SupportsAtomicPartitionManagement with L
case (partitionName, index) =>
val internalRowIndex = partitionSchema.fieldIndex(partitionName)
val structField = partitionSchema.fields(internalRowIndex)
Objects.equals(
// `deepEquals`: a BINARY partition value is a `byte[]`.
Objects.deepEquals(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P1] The new match can delete the NULL partition when the user drops a non-NULL BINARY partition. I reproduced this on Spark 3.5.9 using a native table T (a INT, p BINARY) PARTITIONED BY (p) with (1, CAST(' ' AS BINARY)), (2, NULL), (3, CAST('a' AS BINARY)). ALTER TABLE T DROP PARTITION (p = ' ') succeeds, then SELECT a FROM T returns 1, 3 (expected 2, 3). deepEquals accepts the one-space bytes in the existence check. The commit then converts the same identifier through generatePartValues, maps whitespace to the default partition name, and rebuilds a NULL BinaryRow for the delete. Please carry the typed BINARY identifier through the destructive operation, or reject identifiers that cannot round-trip before committing. Add a regression with both partitions populated.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Reproduced; the STRING version drops the NULL partition on master too. DROP and TRUNCATE now carry the typed identifier through: they match the table's partitions by value, as listPartitionIdentifiers does, and remove the matches by BinaryRow with withOverwriteStaticPartitions. A Hive table with metastore.partitioned-table is dropped through PartitionModification.dropPartitions, which takes partition names, so there a value whose name does not parse back is rejected before anything is removed.

The new tests put ' ', NULL and 'a' in one table, for BINARY and for STRING, and check that dropping ' ' leaves the other two. A Hive catalog test covers the STRING case with and without metastore.partitioned-table.

The same spec made SHOW TABLE EXTENDED ... PARTITION (p = ' ') show the NULL partition's statistics; that lookup now matches by value as well.

sparkInternalRow.get(internalRowIndex, structField.dataType),
internalRow.get(index, structField.dataType))
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,40 @@ abstract class DDLWithHiveCatalogTestBase extends PaimonHiveTestBase {
}
}

test("Paimon DDL with hive catalog: drop a blank partition") {
spark.sql(s"USE $paimonHiveCatalogName")
withDatabase("paimon_db") {
spark.sql("CREATE DATABASE paimon_db")
Seq("false", "true").foreach {
inMetastore =>
withTable("paimon_db.blank_tbl") {
spark.sql(
"CREATE TABLE paimon_db.blank_tbl (a INT, p STRING) USING PAIMON " +
"PARTITIONED BY (p) " +
s"TBLPROPERTIES ('metastore.partitioned-table' = '$inMetastore')")
// In a metastore, blank and NULL have the same partition name, so one commit cannot
// add both.
Seq("(1, ' ')", "(2, NULL)", "(3, 'a')").foreach {
row => spark.sql(s"INSERT INTO paimon_db.blank_tbl VALUES $row")
}
val drop = "ALTER TABLE paimon_db.blank_tbl DROP PARTITION (p = ' ')"
if (inMetastore.toBoolean) {
// The metastore names a blank partition as it names the null one.
val e = intercept[IllegalArgumentException](spark.sql(drop))
assert(e.getMessage.contains("does not parse back"))
assert(!e.getMessage.contains("= NULL"))
checkAnswer(
spark.sql("SELECT a FROM paimon_db.blank_tbl"),
Row(1) :: Row(2) :: Row(3) :: Nil)
} else {
spark.sql(drop)
checkAnswer(spark.sql("SELECT a FROM paimon_db.blank_tbl"), Row(2) :: Row(3) :: Nil)
}
}
}
}
}

test(
"Paimon partition expire test with hive catalog: expire partition for paimon table sparkCatalogName") {
spark.sql(s"USE $paimonHiveCatalogName")
Expand Down
Loading
Loading