From 5059c523676a0133d387b77064a65201652074bb Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 20 Nov 2026 15:55:35 +0800 Subject: [PATCH] [spark] Delete partitions by value in metadata-only DELETE --- .../TruncatePaimonTableWithFilterExec.scala | 76 ++++++++++++------- .../sql/PaimonOptimizationTestBase.scala | 36 +++++++++ 2 files changed, 84 insertions(+), 28 deletions(-) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/TruncatePaimonTableWithFilterExec.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/TruncatePaimonTableWithFilterExec.scala index 3093d15e843e..69781e15850e 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/TruncatePaimonTableWithFilterExec.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/execution/TruncatePaimonTableWithFilterExec.scala @@ -18,15 +18,18 @@ package org.apache.paimon.spark.execution +import org.apache.paimon.{CoreOptions, Snapshot} +import org.apache.paimon.data.BinaryRow +import org.apache.paimon.options.Options import org.apache.paimon.partition.PartitionPredicate import org.apache.paimon.spark.leafnode.PaimonLeafV2CommandExec import org.apache.paimon.table.{FileStoreTable, Table} -import org.apache.paimon.utils.InternalRowPartitionComputer +import org.apache.paimon.table.sink.{BatchTableCommit, CommitMessage} import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.Attribute -import java.util.{Collections => JCollections} +import java.util.{Collections => JCollections, List => JList} import scala.collection.JavaConverters._ @@ -36,37 +39,54 @@ case class TruncatePaimonTableWithFilterExec( extends PaimonLeafV2CommandExec { override def run(): Seq[InternalRow] = { + partitionPredicate match { + case Some(p) => + table match { + case fileStoreTable: FileStoreTable => + val matchedPartitions = + fileStoreTable.newSnapshotReader().withPartitionFilter(p).partitions() + if (matchedPartitions.isEmpty) { + withCommit(table)(_.commit(JCollections.emptyList())) + } else { + truncateMatchedPartitions(fileStoreTable, matchedPartitions) + } + case _ => + throw new UnsupportedOperationException("Unsupported truncate table") + } + case _ => + withCommit(table)(_.truncateTable()) + } + Nil + } + + /** + * Removes the data of the matched partitions by their stored values: an empty static overwrite. A + * spec rebuilt from a stored value does not always parse back to it, such as a blank string, + * whose name is the default partition name. + */ + private def truncateMatchedPartitions( + fileStoreTable: FileStoreTable, + partitions: JList[BinaryRow]): Unit = { + val target = + fileStoreTable.copy(Map(CoreOptions.DYNAMIC_PARTITION_OVERWRITE.key -> "false").asJava) + val commit = target.newCommit(CoreOptions.createCommitUser(new Options(target.options()))) + try { + commit + .withOperation(Snapshot.Operation.TRUNCATE) + .withOverwriteStaticPartitions(partitions) + .commit(JCollections.emptyList[CommitMessage]()) + } finally { + commit.close() + } + } + + private def withCommit(table: Table)(operation: BatchTableCommit => Unit): Unit = { val commit = table.newBatchWriteBuilder().newCommit() try { - partitionPredicate match { - case Some(p) => - table match { - case fileStoreTable: FileStoreTable => - val matchedPartitions = - fileStoreTable.newSnapshotReader().withPartitionFilter(p).partitions().asScala - if (matchedPartitions.nonEmpty) { - val partitionComputer = new InternalRowPartitionComputer( - fileStoreTable.coreOptions().partitionDefaultName(), - fileStoreTable.schema().logicalPartitionType(), - fileStoreTable.partitionKeys.asScala.toArray, - fileStoreTable.coreOptions().legacyPartitionName() - ) - val dropPartitions = - matchedPartitions.map(partitionComputer.generatePartValues(_).asScala.asJava) - commit.truncatePartitions(dropPartitions.asJava) - } else { - commit.commit(JCollections.emptyList()) - } - case _ => - throw new UnsupportedOperationException("Unsupported truncate table") - } - case _ => - commit.truncateTable() - } + operation(commit) } finally { commit.close() } - Nil } override def output: Seq[Attribute] = Nil diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTestBase.scala index 0550e1e4e669..c2090f77ccf4 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonOptimizationTestBase.scala @@ -175,6 +175,42 @@ abstract class PaimonOptimizationTestBase extends PaimonSparkTestBase with Expre } } + test("Paimon Optimization: metadata only delete of partitions whose names do not parse back") { + withTable("t") { + sql("CREATE TABLE t (a INT, p STRING) PARTITIONED BY (p)") + sql("INSERT INTO t VALUES (1, ' '), (2, NULL), (3, 'a')") + // A blank value's partition name is the null partition's. + val df = sql("DELETE FROM t WHERE p = ' '") + checkTruncatePaimonTable(df) + checkAnswer(sql("SELECT a FROM t"), Seq(Row(2), Row(3))) + } + withTable("t") { + sql("CREATE TABLE t (a INT, p1 STRING, p2 BINARY) PARTITIONED BY (p1, p2)") + sql( + "INSERT INTO t VALUES (1, 'x', CAST(' ' AS BINARY)), (2, 'x', NULL), (3, 'y', NULL), " + + "(4, 'x', X'FF')") + // X'FF' is not valid UTF-8, so its partition name holds other bytes. + val df = sql("DELETE FROM t WHERE p1 = 'x'") + checkTruncatePaimonTable(df) + checkAnswer(sql("SELECT a FROM t"), Seq(Row(3))) + } + } + + test("Paimon Optimization: metadata only delete of a timestamp partition off UTC") { + withTimeZone("Asia/Shanghai") { + withTable("t") { + sql("CREATE TABLE t (a INT, p TIMESTAMP) PARTITIONED BY (p)") + // The legacy name of 10:30:00 is its UTC wall clock, which parses back as 02:30:00 here. + sql( + "INSERT INTO t VALUES (1, CAST('2024-06-01 10:30:00' AS TIMESTAMP)), " + + "(2, CAST('2024-06-01 02:30:00' AS TIMESTAMP))") + val df = sql("DELETE FROM t WHERE p = TIMESTAMP '2024-06-01 10:30:00'") + checkTruncatePaimonTable(df) + checkAnswer(sql("SELECT a FROM t"), Seq(Row(2))) + } + } + } + test(s"Paimon Optimization: eval subqueries for delete table with ScalarSubquery") { withPk.foreach( hasPk => {