Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,18 @@

package org.apache.paimon.spark.execution

import org.apache.paimon.{CoreOptions, Snapshot}
import org.apache.paimon.data.BinaryRow
import org.apache.paimon.options.Options
import org.apache.paimon.partition.PartitionPredicate
import org.apache.paimon.spark.leafnode.PaimonLeafV2CommandExec
import org.apache.paimon.table.{FileStoreTable, Table}
import org.apache.paimon.utils.InternalRowPartitionComputer
import org.apache.paimon.table.sink.{BatchTableCommit, CommitMessage}

import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.Attribute

import java.util.{Collections => JCollections}
import java.util.{Collections => JCollections, List => JList}

import scala.collection.JavaConverters._

Expand All @@ -36,37 +39,54 @@ case class TruncatePaimonTableWithFilterExec(
extends PaimonLeafV2CommandExec {

override def run(): Seq[InternalRow] = {
partitionPredicate match {
case Some(p) =>
table match {
case fileStoreTable: FileStoreTable =>
val matchedPartitions =
fileStoreTable.newSnapshotReader().withPartitionFilter(p).partitions()
if (matchedPartitions.isEmpty) {
withCommit(table)(_.commit(JCollections.emptyList()))
} else {
truncateMatchedPartitions(fileStoreTable, matchedPartitions)
}
case _ =>
throw new UnsupportedOperationException("Unsupported truncate table")
}
case _ =>
withCommit(table)(_.truncateTable())
}
Nil
}

/**
* Removes the data of the matched partitions by their stored values: an empty static overwrite. A
* spec rebuilt from a stored value does not always parse back to it, such as a blank string,
* whose name is the default partition name.
*/
private def truncateMatchedPartitions(
fileStoreTable: FileStoreTable,
partitions: JList[BinaryRow]): Unit = {
val target =
fileStoreTable.copy(Map(CoreOptions.DYNAMIC_PARTITION_OVERWRITE.key -> "false").asJava)
val commit = target.newCommit(CoreOptions.createCommitUser(new Options(target.options())))
try {
commit
.withOperation(Snapshot.Operation.TRUNCATE)
.withOverwriteStaticPartitions(partitions)
.commit(JCollections.emptyList[CommitMessage]())
} finally {
commit.close()
}
}

private def withCommit(table: Table)(operation: BatchTableCommit => Unit): Unit = {
val commit = table.newBatchWriteBuilder().newCommit()
try {
partitionPredicate match {
case Some(p) =>
table match {
case fileStoreTable: FileStoreTable =>
val matchedPartitions =
fileStoreTable.newSnapshotReader().withPartitionFilter(p).partitions().asScala
if (matchedPartitions.nonEmpty) {
val partitionComputer = new InternalRowPartitionComputer(
fileStoreTable.coreOptions().partitionDefaultName(),
fileStoreTable.schema().logicalPartitionType(),
fileStoreTable.partitionKeys.asScala.toArray,
fileStoreTable.coreOptions().legacyPartitionName()
)
val dropPartitions =
matchedPartitions.map(partitionComputer.generatePartValues(_).asScala.asJava)
commit.truncatePartitions(dropPartitions.asJava)
} else {
commit.commit(JCollections.emptyList())
}
case _ =>
throw new UnsupportedOperationException("Unsupported truncate table")
}
case _ =>
commit.truncateTable()
}
operation(commit)
} finally {
commit.close()
}
Nil
}

override def output: Seq[Attribute] = Nil
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,42 @@ abstract class PaimonOptimizationTestBase extends PaimonSparkTestBase with Expre
}
}

test("Paimon Optimization: metadata only delete of partitions whose names do not parse back") {
withTable("t") {
sql("CREATE TABLE t (a INT, p STRING) PARTITIONED BY (p)")
sql("INSERT INTO t VALUES (1, ' '), (2, NULL), (3, 'a')")
// A blank value's partition name is the null partition's.
val df = sql("DELETE FROM t WHERE p = ' '")
checkTruncatePaimonTable(df)
checkAnswer(sql("SELECT a FROM t"), Seq(Row(2), Row(3)))
}
withTable("t") {
sql("CREATE TABLE t (a INT, p1 STRING, p2 BINARY) PARTITIONED BY (p1, p2)")
sql(
"INSERT INTO t VALUES (1, 'x', CAST(' ' AS BINARY)), (2, 'x', NULL), (3, 'y', NULL), " +
"(4, 'x', X'FF')")
// X'FF' is not valid UTF-8, so its partition name holds other bytes.
val df = sql("DELETE FROM t WHERE p1 = 'x'")
checkTruncatePaimonTable(df)
checkAnswer(sql("SELECT a FROM t"), Seq(Row(3)))
}
}

test("Paimon Optimization: metadata only delete of a timestamp partition off UTC") {
withTimeZone("Asia/Shanghai") {
withTable("t") {
sql("CREATE TABLE t (a INT, p TIMESTAMP) PARTITIONED BY (p)")
// The legacy name of 10:30:00 is its UTC wall clock, which parses back as 02:30:00 here.
sql(
"INSERT INTO t VALUES (1, CAST('2024-06-01 10:30:00' AS TIMESTAMP)), " +
"(2, CAST('2024-06-01 02:30:00' AS TIMESTAMP))")
val df = sql("DELETE FROM t WHERE p = TIMESTAMP '2024-06-01 10:30:00'")
checkTruncatePaimonTable(df)
checkAnswer(sql("SELECT a FROM t"), Seq(Row(2)))
}
}
}

test(s"Paimon Optimization: eval subqueries for delete table with ScalarSubquery") {
withPk.foreach(
hasPk => {
Expand Down
Loading