diff --git a/docs/docs/multimodal-table/variant.mdx b/docs/docs/multimodal-table/variant.mdx
index bcce4b644aa0..5e773b7fd54e 100644
--- a/docs/docs/multimodal-table/variant.mdx
+++ b/docs/docs/multimodal-table/variant.mdx
@@ -148,7 +148,8 @@ payload (GROUP)
The `typed_value` fields are normal typed Parquet leaves. A reader that pushes Variant path
extractions into the scan can read the required leaves instead of decoding the complete Variant
value. This is most useful when a document is wide but queries repeatedly access a small set of
-paths.
+paths. Paimon's Spark connector does not push these extractions yet; see
+[Query Pushdown](#query-pushdown).
Shredding is lossless:
@@ -246,25 +247,11 @@ Shredding and query pushdown are separate capabilities. Shredding creates typed
the query engine must also push the requested Variant paths into the Paimon scan to avoid reading
the full Variant.
-Spark 4.1 and later can push `variant_get` extractions into Paimon when
-`spark.sql.variant.pushVariantIntoScan` is enabled:
-
-```sql
-SET spark.sql.variant.pushVariantIntoScan = true;
-
-SELECT
- id,
- variant_get(payload, '$.user_id', 'bigint') AS user_id,
- variant_get(payload, '$.city', 'string') AS city
-FROM user_events
-ORDER BY id;
--- (1, 1001, Hangzhou), (2, 1002, Beijing)
-```
-
-The SQL and result are the same for plain, shredded, and mixed-layout files. Shredded files provide
-the largest I/O benefit because the requested paths are available as independent Parquet leaves.
-Selecting the complete Variant, such as `SELECT payload`, requires reading and reconstructing the
-full value.
+The Spark connector does not accept Variant extraction pushdown yet. When a scan accepts an
+extraction, Spark 4.1 and 4.2 build it without column pruning and without its metadata columns,
+which Spark 4.3 fixes in SPARK-57499. The Paimon scan therefore reads the full Variant and Spark
+evaluates `variant_get` on it, whatever `spark.sql.variant.pushVariantIntoScan` is set to. The SQL
+and result are the same for plain, shredded, and mixed-layout files.
## Limitations
diff --git a/paimon-spark/paimon-spark-4.1/src/main/scala/org/apache/paimon/spark/read/PaimonSupportsPushDownVariantExtractions.scala b/paimon-spark/paimon-spark-4.1/src/main/scala/org/apache/paimon/spark/read/PaimonSupportsPushDownVariantExtractions.scala
deleted file mode 100644
index ee27ddf34b03..000000000000
--- a/paimon-spark/paimon-spark-4.1/src/main/scala/org/apache/paimon/spark/read/PaimonSupportsPushDownVariantExtractions.scala
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.spark.read
-
-import org.apache.paimon.spark.SparkTypeUtils
-
-import org.apache.spark.sql.connector.read.{SupportsPushDownVariantExtractions, VariantExtraction}
-import org.apache.spark.sql.execution.datasources.VariantMetadata
-import org.apache.spark.sql.types.VariantType
-
-/** Spark 4.1+ binding; shadows the no-op `paimon-spark-common` trait by FQN. */
-trait PaimonSupportsPushDownVariantExtractions extends SupportsPushDownVariantExtractions {
- protected var acceptedVariantExtractions: Map[Seq[String], Seq[VariantExtractionInfo]] = Map.empty
-
- override def pushVariantExtractions(extractions: Array[VariantExtraction]): Array[Boolean] = {
- val decoded = extractions.iterator.map {
- ex =>
- val vm = VariantMetadata.fromMetadata(ex.metadata())
- val info = VariantExtractionInfo(
- paimonType = SparkTypeUtils.toPaimonType(ex.expectedDataType()),
- path = vm.path,
- failOnError = vm.failOnError,
- timeZoneId = vm.timeZoneId)
- (ex.columnName().toSeq, info, ex.expectedDataType() == VariantType)
- }.toIndexedSeq
- val (newMap, accepted) = VariantPushDownUtils.acceptByPath(decoded)
- acceptedVariantExtractions = newMap
- accepted
- }
-}
diff --git a/paimon-spark/paimon-spark-4.2/pom.xml b/paimon-spark/paimon-spark-4.2/pom.xml
index 5079feb7cb2f..e7d4415ec3c7 100644
--- a/paimon-spark/paimon-spark-4.2/pom.xml
+++ b/paimon-spark/paimon-spark-4.2/pom.xml
@@ -103,6 +103,19 @@ under the License.
spark-hive_${scala.binary.version}
${spark.version}
+
+ org.apache.paimon
+ paimon-spark-ut_${scala.binary.version}
+ ${project.version}
+ tests
+ test
+
+
+ *
+ *
+
+
+
org.apache.spark
spark-sql_${scala.binary.version}
diff --git a/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/VariantTest.scala b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/VariantTest.scala
new file mode 100644
index 000000000000..368020422cf6
--- /dev/null
+++ b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/VariantTest.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.sql
+
+import org.apache.spark.SparkConf
+
+class VariantTest extends VariantTestBase {
+ override protected def sparkConf: SparkConf = {
+ super.sparkConf
+ .set("spark.paimon.variant.inferShreddingSchema", "false")
+ .set("spark.sql.variant.pushVariantIntoScan", "false")
+ }
+}
+
+class VariantInferShreddingTest extends VariantTestBase {
+ override protected def sparkConf: SparkConf = {
+ super.sparkConf
+ .set("spark.paimon.variant.inferShreddingSchema", "true")
+ .set("spark.sql.variant.pushVariantIntoScan", "false")
+ }
+}
+
+class VariantWithPushDownTest extends VariantTestBase {
+ override protected def sparkConf: SparkConf = {
+ super.sparkConf
+ .set("spark.paimon.variant.inferShreddingSchema", "false")
+ .set("spark.sql.variant.pushVariantIntoScan", "true")
+ }
+}
+
+class VariantInferShreddingWithPushDownTest extends VariantTestBase {
+ override protected def sparkConf: SparkConf = {
+ super.sparkConf
+ .set("spark.paimon.variant.inferShreddingSchema", "true")
+ .set("spark.sql.variant.pushVariantIntoScan", "true")
+ }
+}
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/read/PaimonSupportsPushDownVariantExtractions.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/read/PaimonSupportsPushDownVariantExtractions.scala
index 04ff78c5cf99..c9a83be225c2 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/read/PaimonSupportsPushDownVariantExtractions.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/read/PaimonSupportsPushDownVariantExtractions.scala
@@ -19,8 +19,10 @@
package org.apache.paimon.spark.read
/**
- * No-op default for Spark versions without `SupportsPushDownVariantExtractions` (3.x / 4.0). The
- * 4.1 module overrides this trait at the same FQN to mix in the Spark interface. See
+ * No-op on every Spark version: Paimon accepts no variant extractions. On Spark 4.1 and 4.2, once a
+ * scan accepts one, Spark builds it without column pruning and without its metadata columns (fixed
+ * by SPARK-57499 in Spark 4.3). Spark 4.3 still cannot plan a partly accepted call, so a version
+ * module that overrides this trait at the same FQN must accept all extractions or none. See
* [[PaimonSupportsRuntimeFiltering]] for the same shadowing pattern.
*/
trait PaimonSupportsPushDownVariantExtractions {
diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala
index fda16f8219f3..dbe185345c61 100644
--- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala
+++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala
@@ -26,12 +26,13 @@ import org.apache.spark.sql.types.{DataType, StructField, StructType}
abstract class VariantTestBase extends PaimonSparkTestBase {
- private def variantPushDownEnabled: Boolean =
- spark.conf.getOption("spark.sql.variant.pushVariantIntoScan").contains("true")
+ // Paimon accepts no variant extractions (see PaimonSupportsPushDownVariantExtractions), so the
+ // scan reads whole variants even when `spark.sql.variant.pushVariantIntoScan` is on.
+ private def variantPushDownEnabled: Boolean = false
private def scanReadSchemaOf(df: org.apache.spark.sql.DataFrame): StructType = {
df.queryExecution.optimizedPlan
- .collectFirst { case DataSourceV2ScanRelation(_, scan, _, _, _) => scan.readSchema() }
+ .collectFirst { case r: DataSourceV2ScanRelation => r.scan.readSchema() }
.getOrElse(fail("expected a DataSourceV2ScanRelation in the optimized plan"))
}
@@ -1226,7 +1227,7 @@ abstract class VariantTestBase extends PaimonSparkTestBase {
val df = sql("SELECT try_variant_get(v, '$.age', 'int') FROM T")
val desc = df.queryExecution.optimizedPlan
- .collectFirst { case DataSourceV2ScanRelation(_, scan, _, _, _) => scan.description() }
+ .collectFirst { case r: DataSourceV2ScanRelation => r.scan.description() }
.getOrElse(fail("expected a DataSourceV2ScanRelation in the plan"))
if (variantPushDownEnabled) {
@@ -1235,4 +1236,40 @@ abstract class VariantTestBase extends PaimonSparkTestBase {
assert(!desc.contains("PushedVariants"), s"got: $desc")
}
}
+
+ test("Paimon Variant pushdown: extraction next to other variant columns") {
+ sql("CREATE TABLE T (id INT, x INT, v1 VARIANT, v2 VARIANT)")
+ sql("""INSERT INTO T VALUES (1, 10, parse_json('{"a":"x"}'), parse_json('{"b":7}'))""")
+
+ val df = sql("SELECT variant_get(v1, '$.a', 'string') FROM T")
+ val readSchema = scanReadSchemaOf(df)
+ assert(readSchema.fieldNames.toSet == Set("v1"), s"got ${readSchema.fieldNames.toSeq}")
+ checkAnswer(df, Seq(Row("x")))
+ checkAnswer(
+ sql("SELECT variant_get(v1, '$.a', 'string'), variant_get(v2, '$.b', 'int') FROM T"),
+ Seq(Row("x", 7)))
+ }
+
+ test("Paimon Variant pushdown: extraction next to a metadata column") {
+ sql("CREATE TABLE T (id INT, v VARIANT)")
+ sql("""INSERT INTO T VALUES (1, parse_json('{"a":"x"}'))""")
+
+ checkAnswer(
+ sql("SELECT __paimon_file_path IS NOT NULL, variant_get(v, '$.a', 'string') FROM T"),
+ Seq(Row(true, "x")))
+ }
+
+ Seq("true", "false").foreach {
+ dv =>
+ test(s"Paimon Variant pushdown: UPDATE and DELETE with a variant condition, dv=$dv") {
+ sql(
+ "CREATE TABLE T (id INT, v VARIANT) " +
+ s"TBLPROPERTIES ('bucket' = '-1', 'deletion-vectors.enabled' = '$dv')")
+ sql("""INSERT INTO T VALUES (1, parse_json('{"a":"x"}')), (2, parse_json('{"a":"y"}'))""")
+
+ sql("UPDATE T SET id = 10 WHERE variant_get(v, '$.a', 'string') = 'x'")
+ sql("DELETE FROM T WHERE variant_get(v, '$.a', 'string') = 'y'")
+ checkAnswer(sql("SELECT id FROM T"), Seq(Row(10)))
+ }
+ }
}