diff --git a/docs/docs/concepts/rest/semantic-views.md b/docs/docs/concepts/rest/semantic-views.md
index 23af8dc2c9bd..b6ee93c89b6a 100644
--- a/docs/docs/concepts/rest/semantic-views.md
+++ b/docs/docs/concepts/rest/semantic-views.md
@@ -28,7 +28,7 @@ preserves the original document text. Servers validate the model formats, syntax
and features they support; consumers provide query execution.
Registration alone does not grant access to source data or guarantee engine support.
-The Spark 4.2 connector supports the [Ossie query profile](#spark-42-ossie-queries) below.
+The Spark 4.2 and 4.3 connectors support the [Ossie query profile](#spark-42-ossie-queries) below.
This implementation supplies Java clients, wire models, and the
[Catalog OpenAPI contract](/rest-catalog-open-api.yaml). REST server providers must implement
persistence, validation, authorization, and the atomic operations described below.
@@ -168,12 +168,13 @@ Client-side invalid input throws `IllegalArgumentException` before HTTP. An unsu
never falls back to SQL views, labels, or table options. Materialization, rename, and per-measure
or per-field REST resources are outside this version.
-## Spark 4.2 Ossie queries
+## Spark 4.2 and 4.3 Ossie queries {#spark-42-ossie-queries}
-The `paimon-spark-4.2_2.13` connector can query an `ossie-yaml` definition through the configured
-Paimon REST Catalog. Enable `PaimonSparkSessionExtensions` as described in the
-[Spark installation guide](../../spark/installation.mdx). The REST server must implement semantic
-view management; the connector does not provide production REST persistence or validation.
+The `paimon-spark-4.2_2.13` and `paimon-spark-4.3_2.13` connectors can query an `ossie-yaml`
+definition through the configured Paimon REST Catalog. Enable `PaimonSparkSessionExtensions` as
+described in the [Spark installation guide](../../spark/installation.mdx). The REST server must
+implement semantic view management; the connector does not provide production REST persistence or
+validation.
Register a definition whose content is a flat Ossie document, for example:
@@ -237,8 +238,9 @@ register and list definitions. Ordinary tables and SQL views retain their existi
Source table authorization is required independently of permission to read the semantic definition.
When using a CTE, evaluate measures inside its definition (for example,
`WITH m AS (SELECT MEASURE(revenue) AS revenue FROM paimon.sales.order_metrics) SELECT revenue FROM m`).
-Forwarding raw measure columns through a CTE or temporary view is not supported by Spark 4.2's
-Metric View resolver; evaluate measures inside the defining query block before reusing its results.
+Forwarding raw measure columns through a CTE or temporary view is not supported by the Metric View
+resolver of Spark 4.2 and 4.3; evaluate measures inside the defining query block before reusing its
+results.
The bundled schema is pinned to Ossie commit
[`2692b1cc4708fdcf6a91e597ca928a9bc9d0da60`](https://github.com/apache/ossie/tree/2692b1cc4708fdcf6a91e597ca928a9bc9d0da60).
diff --git a/docs/docs/spark/installation.mdx b/docs/docs/spark/installation.mdx
index e223cebe2807..7032587f715b 100644
--- a/docs/docs/spark/installation.mdx
+++ b/docs/docs/spark/installation.mdx
@@ -34,9 +34,9 @@ may require a newer Java runtime. Check its runtime requirements before starting
- Spark 3.x (including 3.5, 3.4, 3.3, 3.2) : Pre-built with Java 8 and Scala 2.12/2.13
-Spark 4.2 is available as an experimental source build with Java 17 and Scala 2.13.
-Its independent `spark42` profile recompiles the shared connector modules against Spark 4.2;
-the `spark4` profile continues to build the Spark 4.0 and 4.1 connectors.
+Spark 4.2 and 4.3 are available as experimental source builds with Java 17 and Scala 2.13.
+Their independent `spark42` and `spark43` profiles recompile the shared connector modules against
+Spark 4.2 and 4.3; the `spark4` profile continues to build the Spark 4.0 and 4.1 connectors.
## Download the Connector
@@ -89,14 +89,18 @@ mvn clean package -DskipTests -pl paimon-spark/paimon-spark-4.1 -am -Pspark4
# build paimon spark 4.2
mvn clean package -DskipTests -pl paimon-spark/paimon-spark-4.2 -am -Pspark42
+
+# build paimon spark 4.3
+mvn clean package -DskipTests -pl paimon-spark/paimon-spark-4.3 -am -Pspark43
```
Use `clean` when switching between Spark profiles: the shared modules produce artifacts with
-the same names, but Spark 4.2 changes internal binary interfaces such as `CatalogManager`.
+the same names, but Spark 4.2 and 4.3 change internal binary interfaces such as `CatalogManager`
+and `ParserInterface`.
For Spark 3.5, you can find the bundled jar in `./paimon-spark/paimon-spark-3.5/target/paimon-spark-3.5_2.12-@@VERSION@@.jar`.
-The Spark 4.2 connector can also query [Ossie Semantic Views](../concepts/rest/semantic-views.md#spark-42-ossie-queries)
+The Spark 4.2 and 4.3 connectors can also query [Ossie Semantic Views](../concepts/rest/semantic-views.md#spark-42-ossie-queries)
registered in a Paimon REST Catalog.
## Add the Connector to Spark
diff --git a/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/Spark42CompatibilityTest.scala b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/Spark42CompatibilityTest.scala
index d535c2f6c275..45d21c3168aa 100644
--- a/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/Spark42CompatibilityTest.scala
+++ b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/Spark42CompatibilityTest.scala
@@ -31,7 +31,10 @@ import org.apache.spark.sql.test.SharedSparkSession
import java.nio.file.Files
import java.util.Collections
-/** Exercise the APIs whose signatures changed in Spark 4.2 using real Paimon files. */
+/**
+ * Exercise the APIs whose signatures changed in Spark 4.2 using real Paimon files. The Spark 4.3
+ * connector compiles and runs this suite too.
+ */
class Spark42CompatibilityTest extends QueryTest with SharedSparkSession {
private val warehouse = Files.createTempDirectory("paimon-spark42-").toFile
private lazy val server = {
@@ -54,6 +57,8 @@ class Spark42CompatibilityTest extends QueryTest with SharedSparkSession {
.set("spark.sql.catalog.paimon.token", "test-token")
.set("spark.sql.defaultCatalog", "paimon")
.set("spark.sql.extensions", classOf[PaimonSparkSessionExtensions].getName)
+ // Spark 4.3 dual-runs the single-pass resolver in tests, and it cannot resolve Paimon views.
+ .set("spark.sql.analyzer.singlePassResolver.dualRunWithLegacy", "false")
override protected def beforeAll(): Unit = {
super.beforeAll()
@@ -86,6 +91,27 @@ class Spark42CompatibilityTest extends QueryTest with SharedSparkSession {
}
}
+ test("create or replace table as select") {
+ withTable("t") {
+ sql("CREATE TABLE t (id INT, label STRING) TBLPROPERTIES ('file.format'='avro')")
+ sql("INSERT INTO t VALUES (1, 'old')")
+ sql(
+ "CREATE OR REPLACE TABLE t TBLPROPERTIES ('file.format'='avro') " +
+ "AS SELECT 2 AS id, 'new' AS label")
+ checkAnswer(sql("SELECT * FROM t"), Seq(Row(2, "new")))
+ }
+ }
+
+ test("replace table with a column list") {
+ withTable("t") {
+ sql("CREATE TABLE t (id INT, label STRING) TBLPROPERTIES ('file.format'='avro')")
+ sql("INSERT INTO t VALUES (1, 'old')")
+ sql("CREATE OR REPLACE TABLE t (id INT, score DOUBLE) TBLPROPERTIES ('file.format'='avro')")
+ sql("INSERT INTO t VALUES (2, 0.5)")
+ checkAnswer(sql("SELECT * FROM t"), Seq(Row(2, 0.5)))
+ }
+ }
+
test("replace table rejects table constraints and keeps the old table") {
withTable("t") {
sql("CREATE TABLE t (id INT) TBLPROPERTIES ('file.format'='avro')")
diff --git a/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala
index f783e4a47014..80fd87caa11f 100644
--- a/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala
+++ b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala
@@ -65,6 +65,8 @@ class OssieSemanticViewTest extends QueryTest with SharedSparkSession {
.set("spark.sql.catalog.paimon.token", "test-token")
.set("spark.sql.catalog.paimon.cache-enabled", "false")
.set("spark.sql.extensions", classOf[PaimonSparkSessionExtensions].getName)
+ // Spark 4.3 dual-runs the single-pass resolver in tests, and it cannot resolve Paimon views.
+ .set("spark.sql.analyzer.singlePassResolver.dualRunWithLegacy", "false")
override protected def beforeAll(): Unit = {
super.beforeAll()
diff --git a/paimon-spark/paimon-spark-4.3/pom.xml b/paimon-spark/paimon-spark-4.3/pom.xml
new file mode 100644
index 000000000000..201e3d3e31f6
--- /dev/null
+++ b/paimon-spark/paimon-spark-4.3/pom.xml
@@ -0,0 +1,219 @@
+
+
+
+ 4.0.0
+
+
+ org.apache.paimon
+ paimon-spark
+ 2.2-SNAPSHOT
+
+
+ paimon-spark-4.3_2.13
+ Paimon : Spark : 4.3 : 2.13
+
+
+
+ 4.3.0
+ 2.13
+ 2.13.18
+ ${project.basedir}/../paimon-spark-4.2
+
+
+
+
+ com.networknt
+ json-schema-validator
+ 2.0.0
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+
+
+ com.fasterxml.jackson.dataformat
+ jackson-dataformat-yaml
+
+
+
+
+ org.yaml
+ snakeyaml
+ 2.5
+ provided
+
+
+ org.apache.paimon
+ paimon-format
+
+
+ org.apache.paimon
+ paimon-spark4-common_${scala.binary.version}
+ ${project.version}
+
+
+ org.apache.spark
+ spark-sql-api_${scala.binary.version}
+ ${spark.version}
+
+
+ org.apache.spark
+ spark-connect-shims_${scala.binary.version}
+
+
+ org.apache.logging.log4j
+ log4j-slf4j2-impl
+
+
+
+
+ org.apache.spark
+ spark-core_${scala.binary.version}
+ ${spark.version}
+
+
+ org.apache.spark
+ spark-sql_${scala.binary.version}
+ ${spark.version}
+
+
+ org.apache.spark
+ spark-catalyst_${scala.binary.version}
+ ${spark.version}
+
+
+ org.apache.spark
+ spark-hive_${scala.binary.version}
+ ${spark.version}
+
+
+ org.apache.spark
+ spark-sql_${scala.binary.version}
+ ${spark.version}
+ tests
+ test
+
+
+ org.apache.spark
+ spark-connect-shims_${scala.binary.version}
+
+
+
+
+ org.apache.spark
+ spark-catalyst_${scala.binary.version}
+ ${spark.version}
+ tests
+ test
+
+
+ org.apache.spark
+ spark-core_${scala.binary.version}
+ ${spark.version}
+ tests
+ test
+
+
+
+
+
+
+ src/main/resources
+
+
+ ${spark42.module.dir}/src/main/resources
+
+ META-INF/NOTICE
+
+
+
+
+
+
+ org.codehaus.mojo
+ build-helper-maven-plugin
+
+
+ add-spark42-sources
+ generate-sources
+
+ add-source
+
+
+
+ ${spark42.module.dir}/src/main/scala
+
+
+
+
+ add-spark42-test-sources
+ generate-test-sources
+
+ add-test-source
+
+
+
+ ${spark42.module.dir}/src/test/scala
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+ shade-paimon
+ package
+
+ shade
+
+
+
+
+ org.apache.paimon:paimon-spark4-common_${scala.binary.version}
+ com.networknt:json-schema-validator
+ com.ethlo.time:itu
+
+
+
+
+ com.networknt
+ org.apache.paimon.spark.shade.networknt
+
+
+ com.ethlo.time
+ org.apache.paimon.spark.shade.time
+
+
+
+
+
+
+
+
+
diff --git a/paimon-spark/paimon-spark-4.3/src/main/resources/META-INF/NOTICE b/paimon-spark/paimon-spark-4.3/src/main/resources/META-INF/NOTICE
new file mode 100644
index 000000000000..e616c59a3f17
--- /dev/null
+++ b/paimon-spark/paimon-spark-4.3/src/main/resources/META-INF/NOTICE
@@ -0,0 +1,12 @@
+paimon-spark-4.3
+Copyright 2023-2026 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt)
+- com.networknt:json-schema-validator:2.0.0
+- com.ethlo.time:itu:1.14.0
+
+The bundled Ossie JSON Schema is from Apache Ossie (incubating), commit
+2692b1cc4708fdcf6a91e597ca928a9bc9d0da60, under the Apache Software License 2.0.
diff --git a/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/sql/Spark43CompatibilityTest.scala b/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/sql/Spark43CompatibilityTest.scala
new file mode 100644
index 000000000000..94921af3e62a
--- /dev/null
+++ b/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/sql/Spark43CompatibilityTest.scala
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.sql
+
+import org.apache.paimon.spark.catalyst.parser.extensions.PaimonSpark4SqlExtensionsParser
+import org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.QueryTest
+import org.apache.spark.sql.test.SharedSparkSession
+
+/** Exercise the APIs that Spark 4.3 added. `Spark42CompatibilityTest` covers the rest. */
+class Spark43CompatibilityTest extends QueryTest with SharedSparkSession {
+
+ override protected def sparkConf: SparkConf = super.sparkConf
+ .set("spark.sql.extensions", classOf[PaimonSparkSessionExtensions].getName)
+
+ test("split statements through the Paimon parser") {
+ val parser = spark.sessionState.sqlParser
+ assert(parser.isInstanceOf[PaimonSpark4SqlExtensionsParser])
+ val result = parser.splitStatements("SELECT ';'; SELECT 2;")
+ assert(result.completeStatements.map(_.statement.trim) === Seq("SELECT ';'", "SELECT 2"))
+ assert(result.partialStatement.isEmpty)
+ }
+}
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/extensions/PaimonSparkSessionExtensions.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/extensions/PaimonSparkSessionExtensions.scala
index e745a560f04b..fe9715cde223 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/extensions/PaimonSparkSessionExtensions.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/extensions/PaimonSparkSessionExtensions.scala
@@ -41,7 +41,7 @@ class PaimonSparkSessionExtensions extends (SparkSessionExtensions => Unit) {
extensions.injectResolutionRule(spark => new PaimonAnalysis(spark))
extensions.injectResolutionRule(spark => PaimonProcedureResolver(spark))
extensions.injectResolutionRule(spark => PaimonViewResolver(spark))
- if (org.apache.spark.SPARK_VERSION.startsWith("4.2.")) {
+ if (org.apache.spark.SPARK_VERSION >= "4.2") {
val resolver = Class
.forName("org.apache.spark.sql.paimon.PaimonSemanticViewResolver")
.getConstructor(classOf[org.apache.spark.sql.SparkSession])
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/PaimonV1FunctionRegistry.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/PaimonV1FunctionRegistry.scala
index 24bf90c99de5..8fbc5a5c6f8f 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/PaimonV1FunctionRegistry.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/PaimonV1FunctionRegistry.scala
@@ -160,13 +160,13 @@ case class PaimonV1FunctionRegistry(session: SparkSession) extends SQLConfHelper
}
val clazz = PaimonUtils.classForName(className)
// The builder name shows up in the default column name of an unaliased call, so leave out
- // the catalog that `qualifyIdentifier` keeps on Spark 4.2.
+ // the catalog that `qualifyIdentifier` keeps on Spark 4.2 and later.
val name = func.identifier.copy(catalog = None).unquotedString
(input) => functionExpressionBuilder.makeExpression(name, clazz, input)
}
private def qualifyIdentifier(ident: FunctionIdentifier): FunctionIdentifier = {
- val catalog = if (org.apache.spark.SPARK_VERSION.startsWith("4.2.")) {
+ val catalog = if (org.apache.spark.SPARK_VERSION >= "4.2") {
ident.catalog.orElse(Some(session.sessionState.catalogManager.currentCatalog.name))
} else None
FunctionIdentifier(
diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonFunctionLookup.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonFunctionLookup.scala
index 7f530d7912ef..b23d15df293f 100644
--- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonFunctionLookup.scala
+++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonFunctionLookup.scala
@@ -54,7 +54,7 @@ case class PaimonFunctionLookup(catalogManager: CatalogManager) extends LookupCa
Some(null, FunctionIdentifier(funName), true)
// Spark 4.2 keeps extension functions in the session's builtin namespace.
case Seq(name)
- if org.apache.spark.SPARK_VERSION.startsWith("4.2.") &&
+ if org.apache.spark.SPARK_VERSION >= "4.2" &&
catalogManager.v1SessionCatalog.lookupBuiltinOrTempFunction(name).isDefined =>
None
case CatalogAndIdentifier(v1FunctionCatalog: SupportV1Function, ident)
diff --git a/paimon-spark/paimon-spark4-common/pom.xml b/paimon-spark/paimon-spark4-common/pom.xml
index 032edb98e032..4bbf62d1f14f 100644
--- a/paimon-spark/paimon-spark4-common/pom.xml
+++ b/paimon-spark/paimon-spark4-common/pom.xml
@@ -153,5 +153,48 @@ under the License.
+
+ spark43
+
+
+
+ com.diffplug.spotless
+ spotless-maven-plugin
+
+
+ src/**/*.scala
+
+
+
+
+ net.alchim31.maven
+ scala-maven-plugin
+
+
+ **/Spark4InternalRow.scala
+ **/Spark4ArrayData.scala
+ **/Spark41*.scala
+ **/PaimonSpark4SqlExtensionsParser.scala
+
+
+
+
+ org.codehaus.mojo
+ build-helper-maven-plugin
+ 3.2.0
+
+
+ spark43-getters
+ generate-sources
+ add-source
+
+ src/main/spark43
+
+
+
+
+
+
+
diff --git a/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala
index 9731b2d22d82..dd5336c36b6a 100644
--- a/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala
+++ b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala
@@ -196,7 +196,7 @@ class Spark4Shim extends SparkShim {
val statementType = "REPLACE TABLE"
val columns = schemaOrColumns.asInstanceOf[Seq[ColumnDefinition]]
ResolveDefaultColumns.validateCatalogForDefaultValue(columns, catalog, ident)
- GeneratedColumn.validateGeneratedColumns(tableSchema, catalog, ident, statementType)
+ validateGeneratedColumns(tableSchema, columns, catalog, ident, statementType)
IdentityColumn.validateIdentityColumn(tableSchema, catalog, ident)
columns.map(_.toV2Column(statementType)).toArray
}
@@ -215,6 +215,27 @@ class Spark4Shim extends SparkShim {
tableSpec.copy(properties = tableSpec.properties ++ additionalProperties, location = location)
}
+ // Spark 4.3 keeps only the catalog check here and verifies generation expressions in
+ // CheckAnalysis.
+ private def validateGeneratedColumns(
+ tableSchema: StructType,
+ columns: Seq[ColumnDefinition],
+ catalog: TableCatalog,
+ ident: Identifier,
+ statementType: String): Unit = {
+ val methods = GeneratedColumn.getClass.getMethods
+ val (method, args) = methods.find(_.getName == "validateGeneratedColumns") match {
+ case Some(m) => (m, Seq(tableSchema, catalog, ident, statementType))
+ case None =>
+ val m = methods
+ .find(_.getName == "validateCatalogForGeneratedColumn")
+ .getOrElse(throw new IllegalStateException("No generated column validation in Spark"))
+ (m, Seq(columns, catalog, ident))
+ }
+ try { method.invoke(GeneratedColumn, args: _*) }
+ catch { case e: java.lang.reflect.InvocationTargetException => throw e.getCause }
+ }
+
private def invalidateCache(tableCatalog: TableCatalog, ident: Identifier): Unit = {
tableCatalog.invalidateTable(ident)
}
@@ -422,7 +443,7 @@ class Spark4Shim extends SparkShim {
readSpatial(array, pos, geography = true)
private def readSpatial(value: AnyRef, pos: Int, geography: Boolean): Array[Byte] = {
- val modern = org.apache.spark.SPARK_VERSION.startsWith("4.2.")
+ val modern = org.apache.spark.SPARK_VERSION >= "4.2"
val getter = if (modern) "getBinaryView" else if (geography) "getGeography" else "getGeometry"
val raw = value.getClass
.getMethod(getter, java.lang.Integer.TYPE)
@@ -447,7 +468,7 @@ class Spark4Shim extends SparkShim {
override def toSparkGeography(wkb: Array[Byte], crs: String, algorithm: String): Object = {
val geographyType = sparkGeographyType(crs, algorithm)
val name =
- if (org.apache.spark.SPARK_VERSION.startsWith("4.2.")) "stGeogSetSrid" else "stSetSrid"
+ if (org.apache.spark.SPARK_VERSION >= "4.2") "stGeogSetSrid" else "stSetSrid"
val raw = STUtils.stGeogFromWKB(wkb)
val method = Class
.forName("org.apache.spark.sql.catalyst.util.STUtils")
diff --git a/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/catalyst/parser/extensions/Spark43SqlExtensionsParser.scala b/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/catalyst/parser/extensions/Spark43SqlExtensionsParser.scala
new file mode 100644
index 000000000000..fb3fa9116896
--- /dev/null
+++ b/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/catalyst/parser/extensions/Spark43SqlExtensionsParser.scala
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.catalyst.parser.extensions
+
+import org.apache.spark.sql.catalyst.parser.{ParserInterface, SqlStatementSplitResult}
+import org.apache.spark.sql.catalyst.parser.extensions.AbstractPaimonSparkSqlExtensionsParser
+import org.apache.spark.sql.types.StructType
+
+/** Spark 4.3 adds `splitStatements` to `ParserInterface`. */
+class PaimonSpark4SqlExtensionsParser(override val delegate: ParserInterface)
+ extends AbstractPaimonSparkSqlExtensionsParser(delegate)
+ with ParserInterface {
+
+ override def parseRoutineParam(sqlText: String): StructType = delegate.parseRoutineParam(sqlText)
+
+ override def splitStatements(sqlText: String): SqlStatementSplitResult =
+ delegate.splitStatements(sqlText)
+}
diff --git a/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/data/Spark43ArrayData.scala b/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/data/Spark43ArrayData.scala
new file mode 100644
index 000000000000..d06f843801ba
--- /dev/null
+++ b/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/data/Spark43ArrayData.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.data
+
+import org.apache.paimon.types.{DataType, GeographyType, GeometryType}
+
+import org.apache.spark.sql.paimon.shims.SparkShimLoader
+import org.apache.spark.unsafe.types.{BinaryView, TimestampNanosVal, VariantVal}
+
+/** Spark 4.3 array getters: BinaryView spatial values, as on 4.2, plus nanosecond timestamps. */
+class Spark4ArrayData(override val elementType: DataType) extends AbstractSparkArrayData {
+ override def getVariant(ordinal: Int): VariantVal = {
+ val value = paimonArray.getVariant(ordinal)
+ new VariantVal(value.value(), value.metadata())
+ }
+
+ override def getBinaryView(ordinal: Int): BinaryView = {
+ val bytes = paimonArray.getBinary(ordinal)
+ val value = elementType match {
+ case geometry: GeometryType =>
+ SparkShimLoader.shim.toSparkGeometry(bytes, geometry.getCrs)
+ case geography: GeographyType =>
+ SparkShimLoader.shim.toSparkGeography(
+ bytes,
+ geography.getCrs,
+ geography.getAlgorithm.toString)
+ case other => throw new UnsupportedOperationException(s"No BinaryView for $other")
+ }
+ value.asInstanceOf[BinaryView]
+ }
+
+ override def getTimestampNTZNanos(ordinal: Int): TimestampNanosVal =
+ throw new UnsupportedOperationException("No Paimon type maps to Spark nanosecond timestamps")
+
+ override def getTimestampLTZNanos(ordinal: Int): TimestampNanosVal =
+ throw new UnsupportedOperationException("No Paimon type maps to Spark nanosecond timestamps")
+}
diff --git a/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/data/Spark43InternalRow.scala b/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/data/Spark43InternalRow.scala
new file mode 100644
index 000000000000..398c4caa46be
--- /dev/null
+++ b/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/data/Spark43InternalRow.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.spark.data
+
+import org.apache.paimon.spark.AbstractSparkInternalRow
+import org.apache.paimon.types.{GeographyType, GeometryType, RowType}
+
+import org.apache.spark.sql.paimon.shims.SparkShimLoader
+import org.apache.spark.unsafe.types.{BinaryView, TimestampNanosVal, VariantVal}
+
+/** Spark 4.3 row getters: BinaryView spatial values, as on 4.2, plus nanosecond timestamps. */
+class Spark4InternalRow(rowType: RowType) extends AbstractSparkInternalRow(rowType) {
+ override def getVariant(i: Int): VariantVal = {
+ val value = row.getVariant(i)
+ new VariantVal(value.value(), value.metadata())
+ }
+
+ override def getBinaryView(ordinal: Int): BinaryView = {
+ val bytes = row.getBinary(ordinal)
+ val value = rowType.getTypeAt(ordinal) match {
+ case geometry: GeometryType =>
+ SparkShimLoader.shim.toSparkGeometry(bytes, geometry.getCrs)
+ case geography: GeographyType =>
+ SparkShimLoader.shim.toSparkGeography(
+ bytes,
+ geography.getCrs,
+ geography.getAlgorithm.toString)
+ case other => throw new UnsupportedOperationException(s"No BinaryView for $other")
+ }
+ value.asInstanceOf[BinaryView]
+ }
+
+ override def getTimestampNTZNanos(ordinal: Int): TimestampNanosVal =
+ throw new UnsupportedOperationException("No Paimon type maps to Spark nanosecond timestamps")
+
+ override def getTimestampLTZNanos(ordinal: Int): TimestampNanosVal =
+ throw new UnsupportedOperationException("No Paimon type maps to Spark nanosecond timestamps")
+}
diff --git a/pom.xml b/pom.xml
index 2fc24798b0e0..598350fa831e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -475,6 +475,25 @@ under the License.
+
+ spark43
+
+ paimon-spark/paimon-spark4-common
+ paimon-spark/paimon-spark-4.3
+
+
+ 17
+ 4.13.1
+ 2.13
+ 2.13.18
+ 4.3.0
+ paimon-spark4-common_2.13
+ 18.1.0
+ 4.3
+ 4.3.0
+
+
+
flink1
diff --git a/tools/ci/plan.py b/tools/ci/plan.py
index e0433ee52e0a..6a01cee53bf1 100644
--- a/tools/ci/plan.py
+++ b/tools/ci/plan.py
@@ -35,6 +35,7 @@
("spark3", "Spark 3 / Scala 2.13", 8, "2.13", 90),
("spark4", "Spark 4 / Scala 2.13", 17, "2.13", 90),
("spark42", "Spark 4.2 / Scala 2.13", 17, "2.13", 90),
+ ("spark43", "Spark 4.3 / Scala 2.13", 17, "2.13", 90),
("e2e-flink1", "E2E / Flink 1 and Spark interoperability", 8, "2.12", 60),
("e2e-flink2", "E2E / Flink 2", 11, "2.12", 60),
("iceberg", "Iceberg GA / JDK 17", 17, "2.12", 60),
@@ -46,7 +47,7 @@
REST_SPECS = {"docs/static/rest-catalog-open-api.yaml", "docs/static/rest-management-open-api.yaml"}
OPENAPI_V4_ACTIONS = "paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java"
FLINK = {"flink1-common", "flink1-connectors", "flink2"}
-SPARK = {"spark3", "spark4", "spark42"}
+SPARK = {"spark3", "spark4", "spark42", "spark43"}
E2E = {"e2e-flink1", "e2e-flink2"}
diff --git a/tools/ci/run-java-tests.sh b/tools/ci/run-java-tests.sh
index eb8061977fe4..30cea6c63876 100644
--- a/tools/ci/run-java-tests.sh
+++ b/tools/ci/run-java-tests.sh
@@ -69,6 +69,10 @@ case "$suite" in
profiles=flink1,spark42
modules=(org.apache.paimon:paimon-spark-4.2_2.13)
;;
+ spark43)
+ profiles=flink1,spark43
+ modules=(org.apache.paimon:paimon-spark-4.3_2.13)
+ ;;
e2e-flink1)
profiles=flink1,spark3,flink-1.20
modules=(paimon-e2e-tests)
diff --git a/tools/ci/test_java_commands.py b/tools/ci/test_java_commands.py
index 346ceccc4fef..05d887264a59 100644
--- a/tools/ci/test_java_commands.py
+++ b/tools/ci/test_java_commands.py
@@ -66,6 +66,9 @@ def test_spark_retains_all_connector_versions_and_scala_variants(self):
spark42 = self.commands('spark42', '2.13', 17)[1]
self.assertEqual(self.modules(spark42), {'org.apache.paimon:paimon-spark-4.2_2.13'})
self.assertIn('-Pflink1,spark42', spark42)
+ spark43 = self.commands('spark43', '2.13', 17)[1]
+ self.assertEqual(self.modules(spark43), {'org.apache.paimon:paimon-spark-4.3_2.13'})
+ self.assertIn('-Pflink1,spark43', spark43)
def test_core_keeps_common_and_docs_tests_and_existing_jdk11_exclusion(self):
for java in (8, 11):
diff --git a/tools/ci/test_plan.py b/tools/ci/test_plan.py
index 89bcf71c6e0f..58990f360c12 100644
--- a/tools/ci/test_plan.py
+++ b/tools/ci/test_plan.py
@@ -35,7 +35,7 @@ def suites(self, paths):
def test_spark_keeps_common_docs_and_interoperability_tests(self):
suites = self.suites(['paimon-spark/paimon-spark-common/src/main/scala/Change.scala'])
- self.assertEqual(suites, {'core', 'spark3', 'spark4', 'spark42', 'e2e-flink1'})
+ self.assertEqual(suites, {'core', 'spark3', 'spark4', 'spark42', 'spark43', 'e2e-flink1'})
def test_flink_keeps_hive_iceberg_and_both_e2e_versions(self):
suites = self.suites(['paimon-flink/paimon-flink-common/src/main/java/Change.java'])
@@ -64,7 +64,7 @@ def test_docs_and_markdown_do_not_start_maven(self):
def test_mixed_changes_take_union(self):
self.assertEqual(self.suites(['docs/intro.md', 'paimon-spark/pom.xml',
'paimon-flink/pom.xml']),
- {'core', 'spark3', 'spark4', 'spark42', 'flink1-common',
+ {'core', 'spark3', 'spark4', 'spark42', 'spark43', 'flink1-common',
'flink1-connectors', 'flink2', 'e2e-flink1',
'e2e-flink2', 'iceberg'})
result = plan.select(['paimon-python/setup.py', 'paimon-core/Change.java'])
@@ -220,7 +220,7 @@ def test_pr_size_guard_ignores_unchanged_but_rejects_changed_oversized_files(sel
base = self.git('rev-parse', 'HEAD')
self.commit('paimon-spark/Change.scala', 'spark')
outputs, _ = self.run_pr_plan(base, self.git('rev-parse', 'HEAD'))
- self.assertEqual(len(outputs['matrix']['include']), 7)
+ self.assertEqual(len(outputs['matrix']['include']), 8)
for path in ('docs/static/old image.png', 'docs/static/new image.png'):
with self.subTest(path=path):
@@ -241,7 +241,7 @@ def test_github_outputs_are_valid_json_and_summary_explains_selection(self):
outputs = dict(line.split('=', 1) for line in Path('output.txt').read_text().splitlines())
self.assertTrue(json.loads(outputs['java']))
self.assertFalse(json.loads(outputs['python']))
- self.assertEqual(len(json.loads(outputs['matrix'])['include']), 7)
+ self.assertEqual(len(json.loads(outputs['matrix'])['include']), 8)
self.assertIn('Spark interoperability', Path('summary.md').read_text())