From 2165deb497780ba25e7c5665005914a56fe0cf38 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 20 Nov 2026 14:38:05 +0800 Subject: [PATCH] [spark] Stop accepting variant extraction pushdown on Spark 4.1 and 4.2 --- docs/docs/multimodal-table/variant.mdx | 27 +++------- ...onSupportsPushDownVariantExtractions.scala | 46 ---------------- paimon-spark/paimon-spark-4.2/pom.xml | 13 +++++ .../apache/paimon/spark/sql/VariantTest.scala | 53 +++++++++++++++++++ ...onSupportsPushDownVariantExtractions.scala | 6 ++- .../paimon/spark/sql/VariantTestBase.scala | 45 ++++++++++++++-- 6 files changed, 118 insertions(+), 72 deletions(-) delete mode 100644 paimon-spark/paimon-spark-4.1/src/main/scala/org/apache/paimon/spark/read/PaimonSupportsPushDownVariantExtractions.scala create mode 100644 paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/VariantTest.scala diff --git a/docs/docs/multimodal-table/variant.mdx b/docs/docs/multimodal-table/variant.mdx index bcce4b644aa0..5e773b7fd54e 100644 --- a/docs/docs/multimodal-table/variant.mdx +++ b/docs/docs/multimodal-table/variant.mdx @@ -148,7 +148,8 @@ payload (GROUP) The `typed_value` fields are normal typed Parquet leaves. A reader that pushes Variant path extractions into the scan can read the required leaves instead of decoding the complete Variant value. This is most useful when a document is wide but queries repeatedly access a small set of -paths. +paths. Paimon's Spark connector does not push these extractions yet; see +[Query Pushdown](#query-pushdown). Shredding is lossless: @@ -246,25 +247,11 @@ Shredding and query pushdown are separate capabilities. Shredding creates typed the query engine must also push the requested Variant paths into the Paimon scan to avoid reading the full Variant. -Spark 4.1 and later can push `variant_get` extractions into Paimon when -`spark.sql.variant.pushVariantIntoScan` is enabled: - -```sql -SET spark.sql.variant.pushVariantIntoScan = true; - -SELECT - id, - variant_get(payload, '$.user_id', 'bigint') AS user_id, - variant_get(payload, '$.city', 'string') AS city -FROM user_events -ORDER BY id; --- (1, 1001, Hangzhou), (2, 1002, Beijing) -``` - -The SQL and result are the same for plain, shredded, and mixed-layout files. Shredded files provide -the largest I/O benefit because the requested paths are available as independent Parquet leaves. -Selecting the complete Variant, such as `SELECT payload`, requires reading and reconstructing the -full value. +The Spark connector does not accept Variant extraction pushdown yet. When a scan accepts an +extraction, Spark 4.1 and 4.2 build it without column pruning and without its metadata columns, +which Spark 4.3 fixes in SPARK-57499. The Paimon scan therefore reads the full Variant and Spark +evaluates `variant_get` on it, whatever `spark.sql.variant.pushVariantIntoScan` is set to. The SQL +and result are the same for plain, shredded, and mixed-layout files. ## Limitations diff --git a/paimon-spark/paimon-spark-4.1/src/main/scala/org/apache/paimon/spark/read/PaimonSupportsPushDownVariantExtractions.scala b/paimon-spark/paimon-spark-4.1/src/main/scala/org/apache/paimon/spark/read/PaimonSupportsPushDownVariantExtractions.scala deleted file mode 100644 index ee27ddf34b03..000000000000 --- a/paimon-spark/paimon-spark-4.1/src/main/scala/org/apache/paimon/spark/read/PaimonSupportsPushDownVariantExtractions.scala +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.spark.read - -import org.apache.paimon.spark.SparkTypeUtils - -import org.apache.spark.sql.connector.read.{SupportsPushDownVariantExtractions, VariantExtraction} -import org.apache.spark.sql.execution.datasources.VariantMetadata -import org.apache.spark.sql.types.VariantType - -/** Spark 4.1+ binding; shadows the no-op `paimon-spark-common` trait by FQN. */ -trait PaimonSupportsPushDownVariantExtractions extends SupportsPushDownVariantExtractions { - protected var acceptedVariantExtractions: Map[Seq[String], Seq[VariantExtractionInfo]] = Map.empty - - override def pushVariantExtractions(extractions: Array[VariantExtraction]): Array[Boolean] = { - val decoded = extractions.iterator.map { - ex => - val vm = VariantMetadata.fromMetadata(ex.metadata()) - val info = VariantExtractionInfo( - paimonType = SparkTypeUtils.toPaimonType(ex.expectedDataType()), - path = vm.path, - failOnError = vm.failOnError, - timeZoneId = vm.timeZoneId) - (ex.columnName().toSeq, info, ex.expectedDataType() == VariantType) - }.toIndexedSeq - val (newMap, accepted) = VariantPushDownUtils.acceptByPath(decoded) - acceptedVariantExtractions = newMap - accepted - } -} diff --git a/paimon-spark/paimon-spark-4.2/pom.xml b/paimon-spark/paimon-spark-4.2/pom.xml index 5079feb7cb2f..e7d4415ec3c7 100644 --- a/paimon-spark/paimon-spark-4.2/pom.xml +++ b/paimon-spark/paimon-spark-4.2/pom.xml @@ -103,6 +103,19 @@ under the License. spark-hive_${scala.binary.version} ${spark.version} + + org.apache.paimon + paimon-spark-ut_${scala.binary.version} + ${project.version} + tests + test + + + * + * + + + org.apache.spark spark-sql_${scala.binary.version} diff --git a/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/VariantTest.scala b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/VariantTest.scala new file mode 100644 index 000000000000..368020422cf6 --- /dev/null +++ b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/VariantTest.scala @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +import org.apache.spark.SparkConf + +class VariantTest extends VariantTestBase { + override protected def sparkConf: SparkConf = { + super.sparkConf + .set("spark.paimon.variant.inferShreddingSchema", "false") + .set("spark.sql.variant.pushVariantIntoScan", "false") + } +} + +class VariantInferShreddingTest extends VariantTestBase { + override protected def sparkConf: SparkConf = { + super.sparkConf + .set("spark.paimon.variant.inferShreddingSchema", "true") + .set("spark.sql.variant.pushVariantIntoScan", "false") + } +} + +class VariantWithPushDownTest extends VariantTestBase { + override protected def sparkConf: SparkConf = { + super.sparkConf + .set("spark.paimon.variant.inferShreddingSchema", "false") + .set("spark.sql.variant.pushVariantIntoScan", "true") + } +} + +class VariantInferShreddingWithPushDownTest extends VariantTestBase { + override protected def sparkConf: SparkConf = { + super.sparkConf + .set("spark.paimon.variant.inferShreddingSchema", "true") + .set("spark.sql.variant.pushVariantIntoScan", "true") + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/read/PaimonSupportsPushDownVariantExtractions.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/read/PaimonSupportsPushDownVariantExtractions.scala index 04ff78c5cf99..c9a83be225c2 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/read/PaimonSupportsPushDownVariantExtractions.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/read/PaimonSupportsPushDownVariantExtractions.scala @@ -19,8 +19,10 @@ package org.apache.paimon.spark.read /** - * No-op default for Spark versions without `SupportsPushDownVariantExtractions` (3.x / 4.0). The - * 4.1 module overrides this trait at the same FQN to mix in the Spark interface. See + * No-op on every Spark version: Paimon accepts no variant extractions. On Spark 4.1 and 4.2, once a + * scan accepts one, Spark builds it without column pruning and without its metadata columns (fixed + * by SPARK-57499 in Spark 4.3). Spark 4.3 still cannot plan a partly accepted call, so a version + * module that overrides this trait at the same FQN must accept all extractions or none. See * [[PaimonSupportsRuntimeFiltering]] for the same shadowing pattern. */ trait PaimonSupportsPushDownVariantExtractions { diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala index fda16f8219f3..dbe185345c61 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala @@ -26,12 +26,13 @@ import org.apache.spark.sql.types.{DataType, StructField, StructType} abstract class VariantTestBase extends PaimonSparkTestBase { - private def variantPushDownEnabled: Boolean = - spark.conf.getOption("spark.sql.variant.pushVariantIntoScan").contains("true") + // Paimon accepts no variant extractions (see PaimonSupportsPushDownVariantExtractions), so the + // scan reads whole variants even when `spark.sql.variant.pushVariantIntoScan` is on. + private def variantPushDownEnabled: Boolean = false private def scanReadSchemaOf(df: org.apache.spark.sql.DataFrame): StructType = { df.queryExecution.optimizedPlan - .collectFirst { case DataSourceV2ScanRelation(_, scan, _, _, _) => scan.readSchema() } + .collectFirst { case r: DataSourceV2ScanRelation => r.scan.readSchema() } .getOrElse(fail("expected a DataSourceV2ScanRelation in the optimized plan")) } @@ -1226,7 +1227,7 @@ abstract class VariantTestBase extends PaimonSparkTestBase { val df = sql("SELECT try_variant_get(v, '$.age', 'int') FROM T") val desc = df.queryExecution.optimizedPlan - .collectFirst { case DataSourceV2ScanRelation(_, scan, _, _, _) => scan.description() } + .collectFirst { case r: DataSourceV2ScanRelation => r.scan.description() } .getOrElse(fail("expected a DataSourceV2ScanRelation in the plan")) if (variantPushDownEnabled) { @@ -1235,4 +1236,40 @@ abstract class VariantTestBase extends PaimonSparkTestBase { assert(!desc.contains("PushedVariants"), s"got: $desc") } } + + test("Paimon Variant pushdown: extraction next to other variant columns") { + sql("CREATE TABLE T (id INT, x INT, v1 VARIANT, v2 VARIANT)") + sql("""INSERT INTO T VALUES (1, 10, parse_json('{"a":"x"}'), parse_json('{"b":7}'))""") + + val df = sql("SELECT variant_get(v1, '$.a', 'string') FROM T") + val readSchema = scanReadSchemaOf(df) + assert(readSchema.fieldNames.toSet == Set("v1"), s"got ${readSchema.fieldNames.toSeq}") + checkAnswer(df, Seq(Row("x"))) + checkAnswer( + sql("SELECT variant_get(v1, '$.a', 'string'), variant_get(v2, '$.b', 'int') FROM T"), + Seq(Row("x", 7))) + } + + test("Paimon Variant pushdown: extraction next to a metadata column") { + sql("CREATE TABLE T (id INT, v VARIANT)") + sql("""INSERT INTO T VALUES (1, parse_json('{"a":"x"}'))""") + + checkAnswer( + sql("SELECT __paimon_file_path IS NOT NULL, variant_get(v, '$.a', 'string') FROM T"), + Seq(Row(true, "x"))) + } + + Seq("true", "false").foreach { + dv => + test(s"Paimon Variant pushdown: UPDATE and DELETE with a variant condition, dv=$dv") { + sql( + "CREATE TABLE T (id INT, v VARIANT) " + + s"TBLPROPERTIES ('bucket' = '-1', 'deletion-vectors.enabled' = '$dv')") + sql("""INSERT INTO T VALUES (1, parse_json('{"a":"x"}')), (2, parse_json('{"a":"y"}'))""") + + sql("UPDATE T SET id = 10 WHERE variant_get(v, '$.a', 'string') = 'x'") + sql("DELETE FROM T WHERE variant_get(v, '$.a', 'string') = 'y'") + checkAnswer(sql("SELECT id FROM T"), Seq(Row(10))) + } + } }