diff --git a/docs/docs/concepts/rest/semantic-views.md b/docs/docs/concepts/rest/semantic-views.md index 23af8dc2c9bd..b6ee93c89b6a 100644 --- a/docs/docs/concepts/rest/semantic-views.md +++ b/docs/docs/concepts/rest/semantic-views.md @@ -28,7 +28,7 @@ preserves the original document text. Servers validate the model formats, syntax and features they support; consumers provide query execution. Registration alone does not grant access to source data or guarantee engine support. -The Spark 4.2 connector supports the [Ossie query profile](#spark-42-ossie-queries) below. +The Spark 4.2 and 4.3 connectors support the [Ossie query profile](#spark-42-ossie-queries) below. This implementation supplies Java clients, wire models, and the [Catalog OpenAPI contract](/rest-catalog-open-api.yaml). REST server providers must implement persistence, validation, authorization, and the atomic operations described below. @@ -168,12 +168,13 @@ Client-side invalid input throws `IllegalArgumentException` before HTTP. An unsu never falls back to SQL views, labels, or table options. Materialization, rename, and per-measure or per-field REST resources are outside this version. -## Spark 4.2 Ossie queries +## Spark 4.2 and 4.3 Ossie queries {#spark-42-ossie-queries} -The `paimon-spark-4.2_2.13` connector can query an `ossie-yaml` definition through the configured -Paimon REST Catalog. Enable `PaimonSparkSessionExtensions` as described in the -[Spark installation guide](../../spark/installation.mdx). The REST server must implement semantic -view management; the connector does not provide production REST persistence or validation. +The `paimon-spark-4.2_2.13` and `paimon-spark-4.3_2.13` connectors can query an `ossie-yaml` +definition through the configured Paimon REST Catalog. Enable `PaimonSparkSessionExtensions` as +described in the [Spark installation guide](../../spark/installation.mdx). The REST server must +implement semantic view management; the connector does not provide production REST persistence or +validation. Register a definition whose content is a flat Ossie document, for example: @@ -237,8 +238,9 @@ register and list definitions. Ordinary tables and SQL views retain their existi Source table authorization is required independently of permission to read the semantic definition. When using a CTE, evaluate measures inside its definition (for example, `WITH m AS (SELECT MEASURE(revenue) AS revenue FROM paimon.sales.order_metrics) SELECT revenue FROM m`). -Forwarding raw measure columns through a CTE or temporary view is not supported by Spark 4.2's -Metric View resolver; evaluate measures inside the defining query block before reusing its results. +Forwarding raw measure columns through a CTE or temporary view is not supported by the Metric View +resolver of Spark 4.2 and 4.3; evaluate measures inside the defining query block before reusing its +results. The bundled schema is pinned to Ossie commit [`2692b1cc4708fdcf6a91e597ca928a9bc9d0da60`](https://github.com/apache/ossie/tree/2692b1cc4708fdcf6a91e597ca928a9bc9d0da60). diff --git a/docs/docs/spark/installation.mdx b/docs/docs/spark/installation.mdx index e223cebe2807..7032587f715b 100644 --- a/docs/docs/spark/installation.mdx +++ b/docs/docs/spark/installation.mdx @@ -34,9 +34,9 @@ may require a newer Java runtime. Check its runtime requirements before starting - Spark 3.x (including 3.5, 3.4, 3.3, 3.2) : Pre-built with Java 8 and Scala 2.12/2.13 -Spark 4.2 is available as an experimental source build with Java 17 and Scala 2.13. -Its independent `spark42` profile recompiles the shared connector modules against Spark 4.2; -the `spark4` profile continues to build the Spark 4.0 and 4.1 connectors. +Spark 4.2 and 4.3 are available as experimental source builds with Java 17 and Scala 2.13. +Their independent `spark42` and `spark43` profiles recompile the shared connector modules against +Spark 4.2 and 4.3; the `spark4` profile continues to build the Spark 4.0 and 4.1 connectors. ## Download the Connector @@ -89,14 +89,18 @@ mvn clean package -DskipTests -pl paimon-spark/paimon-spark-4.1 -am -Pspark4 # build paimon spark 4.2 mvn clean package -DskipTests -pl paimon-spark/paimon-spark-4.2 -am -Pspark42 + +# build paimon spark 4.3 +mvn clean package -DskipTests -pl paimon-spark/paimon-spark-4.3 -am -Pspark43 ``` Use `clean` when switching between Spark profiles: the shared modules produce artifacts with -the same names, but Spark 4.2 changes internal binary interfaces such as `CatalogManager`. +the same names, but Spark 4.2 and 4.3 change internal binary interfaces such as `CatalogManager` +and `ParserInterface`. For Spark 3.5, you can find the bundled jar in `./paimon-spark/paimon-spark-3.5/target/paimon-spark-3.5_2.12-@@VERSION@@.jar`. -The Spark 4.2 connector can also query [Ossie Semantic Views](../concepts/rest/semantic-views.md#spark-42-ossie-queries) +The Spark 4.2 and 4.3 connectors can also query [Ossie Semantic Views](../concepts/rest/semantic-views.md#spark-42-ossie-queries) registered in a Paimon REST Catalog. ## Add the Connector to Spark diff --git a/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/Spark42CompatibilityTest.scala b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/Spark42CompatibilityTest.scala index d535c2f6c275..45d21c3168aa 100644 --- a/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/Spark42CompatibilityTest.scala +++ b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/Spark42CompatibilityTest.scala @@ -31,7 +31,10 @@ import org.apache.spark.sql.test.SharedSparkSession import java.nio.file.Files import java.util.Collections -/** Exercise the APIs whose signatures changed in Spark 4.2 using real Paimon files. */ +/** + * Exercise the APIs whose signatures changed in Spark 4.2 using real Paimon files. The Spark 4.3 + * connector compiles and runs this suite too. + */ class Spark42CompatibilityTest extends QueryTest with SharedSparkSession { private val warehouse = Files.createTempDirectory("paimon-spark42-").toFile private lazy val server = { @@ -54,6 +57,8 @@ class Spark42CompatibilityTest extends QueryTest with SharedSparkSession { .set("spark.sql.catalog.paimon.token", "test-token") .set("spark.sql.defaultCatalog", "paimon") .set("spark.sql.extensions", classOf[PaimonSparkSessionExtensions].getName) + // Spark 4.3 dual-runs the single-pass resolver in tests, and it cannot resolve Paimon views. + .set("spark.sql.analyzer.singlePassResolver.dualRunWithLegacy", "false") override protected def beforeAll(): Unit = { super.beforeAll() @@ -86,6 +91,27 @@ class Spark42CompatibilityTest extends QueryTest with SharedSparkSession { } } + test("create or replace table as select") { + withTable("t") { + sql("CREATE TABLE t (id INT, label STRING) TBLPROPERTIES ('file.format'='avro')") + sql("INSERT INTO t VALUES (1, 'old')") + sql( + "CREATE OR REPLACE TABLE t TBLPROPERTIES ('file.format'='avro') " + + "AS SELECT 2 AS id, 'new' AS label") + checkAnswer(sql("SELECT * FROM t"), Seq(Row(2, "new"))) + } + } + + test("replace table with a column list") { + withTable("t") { + sql("CREATE TABLE t (id INT, label STRING) TBLPROPERTIES ('file.format'='avro')") + sql("INSERT INTO t VALUES (1, 'old')") + sql("CREATE OR REPLACE TABLE t (id INT, score DOUBLE) TBLPROPERTIES ('file.format'='avro')") + sql("INSERT INTO t VALUES (2, 0.5)") + checkAnswer(sql("SELECT * FROM t"), Seq(Row(2, 0.5))) + } + } + test("replace table rejects table constraints and keeps the old table") { withTable("t") { sql("CREATE TABLE t (id INT) TBLPROPERTIES ('file.format'='avro')") diff --git a/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala index f783e4a47014..80fd87caa11f 100644 --- a/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala +++ b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala @@ -65,6 +65,8 @@ class OssieSemanticViewTest extends QueryTest with SharedSparkSession { .set("spark.sql.catalog.paimon.token", "test-token") .set("spark.sql.catalog.paimon.cache-enabled", "false") .set("spark.sql.extensions", classOf[PaimonSparkSessionExtensions].getName) + // Spark 4.3 dual-runs the single-pass resolver in tests, and it cannot resolve Paimon views. + .set("spark.sql.analyzer.singlePassResolver.dualRunWithLegacy", "false") override protected def beforeAll(): Unit = { super.beforeAll() diff --git a/paimon-spark/paimon-spark-4.3/pom.xml b/paimon-spark/paimon-spark-4.3/pom.xml new file mode 100644 index 000000000000..201e3d3e31f6 --- /dev/null +++ b/paimon-spark/paimon-spark-4.3/pom.xml @@ -0,0 +1,219 @@ + + + + 4.0.0 + + + org.apache.paimon + paimon-spark + 2.2-SNAPSHOT + + + paimon-spark-4.3_2.13 + Paimon : Spark : 4.3 : 2.13 + + + + 4.3.0 + 2.13 + 2.13.18 + ${project.basedir}/../paimon-spark-4.2 + + + + + com.networknt + json-schema-validator + 2.0.0 + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + + + org.yaml + snakeyaml + 2.5 + provided + + + org.apache.paimon + paimon-format + + + org.apache.paimon + paimon-spark4-common_${scala.binary.version} + ${project.version} + + + org.apache.spark + spark-sql-api_${scala.binary.version} + ${spark.version} + + + org.apache.spark + spark-connect-shims_${scala.binary.version} + + + org.apache.logging.log4j + log4j-slf4j2-impl + + + + + org.apache.spark + spark-core_${scala.binary.version} + ${spark.version} + + + org.apache.spark + spark-sql_${scala.binary.version} + ${spark.version} + + + org.apache.spark + spark-catalyst_${scala.binary.version} + ${spark.version} + + + org.apache.spark + spark-hive_${scala.binary.version} + ${spark.version} + + + org.apache.spark + spark-sql_${scala.binary.version} + ${spark.version} + tests + test + + + org.apache.spark + spark-connect-shims_${scala.binary.version} + + + + + org.apache.spark + spark-catalyst_${scala.binary.version} + ${spark.version} + tests + test + + + org.apache.spark + spark-core_${scala.binary.version} + ${spark.version} + tests + test + + + + + + + src/main/resources + + + ${spark42.module.dir}/src/main/resources + + META-INF/NOTICE + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-spark42-sources + generate-sources + + add-source + + + + ${spark42.module.dir}/src/main/scala + + + + + add-spark42-test-sources + generate-test-sources + + add-test-source + + + + ${spark42.module.dir}/src/test/scala + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-paimon + package + + shade + + + + + org.apache.paimon:paimon-spark4-common_${scala.binary.version} + com.networknt:json-schema-validator + com.ethlo.time:itu + + + + + com.networknt + org.apache.paimon.spark.shade.networknt + + + com.ethlo.time + org.apache.paimon.spark.shade.time + + + + + + + + + diff --git a/paimon-spark/paimon-spark-4.3/src/main/resources/META-INF/NOTICE b/paimon-spark/paimon-spark-4.3/src/main/resources/META-INF/NOTICE new file mode 100644 index 000000000000..e616c59a3f17 --- /dev/null +++ b/paimon-spark/paimon-spark-4.3/src/main/resources/META-INF/NOTICE @@ -0,0 +1,12 @@ +paimon-spark-4.3 +Copyright 2023-2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) +- com.networknt:json-schema-validator:2.0.0 +- com.ethlo.time:itu:1.14.0 + +The bundled Ossie JSON Schema is from Apache Ossie (incubating), commit +2692b1cc4708fdcf6a91e597ca928a9bc9d0da60, under the Apache Software License 2.0. diff --git a/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/sql/Spark43CompatibilityTest.scala b/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/sql/Spark43CompatibilityTest.scala new file mode 100644 index 000000000000..94921af3e62a --- /dev/null +++ b/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/sql/Spark43CompatibilityTest.scala @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +import org.apache.paimon.spark.catalyst.parser.extensions.PaimonSpark4SqlExtensionsParser +import org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions + +import org.apache.spark.SparkConf +import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.test.SharedSparkSession + +/** Exercise the APIs that Spark 4.3 added. `Spark42CompatibilityTest` covers the rest. */ +class Spark43CompatibilityTest extends QueryTest with SharedSparkSession { + + override protected def sparkConf: SparkConf = super.sparkConf + .set("spark.sql.extensions", classOf[PaimonSparkSessionExtensions].getName) + + test("split statements through the Paimon parser") { + val parser = spark.sessionState.sqlParser + assert(parser.isInstanceOf[PaimonSpark4SqlExtensionsParser]) + val result = parser.splitStatements("SELECT ';'; SELECT 2;") + assert(result.completeStatements.map(_.statement.trim) === Seq("SELECT ';'", "SELECT 2")) + assert(result.partialStatement.isEmpty) + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/extensions/PaimonSparkSessionExtensions.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/extensions/PaimonSparkSessionExtensions.scala index e745a560f04b..fe9715cde223 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/extensions/PaimonSparkSessionExtensions.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/extensions/PaimonSparkSessionExtensions.scala @@ -41,7 +41,7 @@ class PaimonSparkSessionExtensions extends (SparkSessionExtensions => Unit) { extensions.injectResolutionRule(spark => new PaimonAnalysis(spark)) extensions.injectResolutionRule(spark => PaimonProcedureResolver(spark)) extensions.injectResolutionRule(spark => PaimonViewResolver(spark)) - if (org.apache.spark.SPARK_VERSION.startsWith("4.2.")) { + if (org.apache.spark.SPARK_VERSION >= "4.2") { val resolver = Class .forName("org.apache.spark.sql.paimon.PaimonSemanticViewResolver") .getConstructor(classOf[org.apache.spark.sql.SparkSession]) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/PaimonV1FunctionRegistry.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/PaimonV1FunctionRegistry.scala index 24bf90c99de5..8fbc5a5c6f8f 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/PaimonV1FunctionRegistry.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/PaimonV1FunctionRegistry.scala @@ -160,13 +160,13 @@ case class PaimonV1FunctionRegistry(session: SparkSession) extends SQLConfHelper } val clazz = PaimonUtils.classForName(className) // The builder name shows up in the default column name of an unaliased call, so leave out - // the catalog that `qualifyIdentifier` keeps on Spark 4.2. + // the catalog that `qualifyIdentifier` keeps on Spark 4.2 and later. val name = func.identifier.copy(catalog = None).unquotedString (input) => functionExpressionBuilder.makeExpression(name, clazz, input) } private def qualifyIdentifier(ident: FunctionIdentifier): FunctionIdentifier = { - val catalog = if (org.apache.spark.SPARK_VERSION.startsWith("4.2.")) { + val catalog = if (org.apache.spark.SPARK_VERSION >= "4.2") { ident.catalog.orElse(Some(session.sessionState.catalogManager.currentCatalog.name)) } else None FunctionIdentifier( diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonFunctionLookup.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonFunctionLookup.scala index 7f530d7912ef..b23d15df293f 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonFunctionLookup.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonFunctionLookup.scala @@ -54,7 +54,7 @@ case class PaimonFunctionLookup(catalogManager: CatalogManager) extends LookupCa Some(null, FunctionIdentifier(funName), true) // Spark 4.2 keeps extension functions in the session's builtin namespace. case Seq(name) - if org.apache.spark.SPARK_VERSION.startsWith("4.2.") && + if org.apache.spark.SPARK_VERSION >= "4.2" && catalogManager.v1SessionCatalog.lookupBuiltinOrTempFunction(name).isDefined => None case CatalogAndIdentifier(v1FunctionCatalog: SupportV1Function, ident) diff --git a/paimon-spark/paimon-spark4-common/pom.xml b/paimon-spark/paimon-spark4-common/pom.xml index 032edb98e032..4bbf62d1f14f 100644 --- a/paimon-spark/paimon-spark4-common/pom.xml +++ b/paimon-spark/paimon-spark4-common/pom.xml @@ -153,5 +153,48 @@ under the License. + + spark43 + + + + com.diffplug.spotless + spotless-maven-plugin + + + src/**/*.scala + + + + + net.alchim31.maven + scala-maven-plugin + + + **/Spark4InternalRow.scala + **/Spark4ArrayData.scala + **/Spark41*.scala + **/PaimonSpark4SqlExtensionsParser.scala + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.2.0 + + + spark43-getters + generate-sources + add-source + + src/main/spark43 + + + + + + + diff --git a/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala index 9731b2d22d82..dd5336c36b6a 100644 --- a/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala +++ b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala @@ -196,7 +196,7 @@ class Spark4Shim extends SparkShim { val statementType = "REPLACE TABLE" val columns = schemaOrColumns.asInstanceOf[Seq[ColumnDefinition]] ResolveDefaultColumns.validateCatalogForDefaultValue(columns, catalog, ident) - GeneratedColumn.validateGeneratedColumns(tableSchema, catalog, ident, statementType) + validateGeneratedColumns(tableSchema, columns, catalog, ident, statementType) IdentityColumn.validateIdentityColumn(tableSchema, catalog, ident) columns.map(_.toV2Column(statementType)).toArray } @@ -215,6 +215,27 @@ class Spark4Shim extends SparkShim { tableSpec.copy(properties = tableSpec.properties ++ additionalProperties, location = location) } + // Spark 4.3 keeps only the catalog check here and verifies generation expressions in + // CheckAnalysis. + private def validateGeneratedColumns( + tableSchema: StructType, + columns: Seq[ColumnDefinition], + catalog: TableCatalog, + ident: Identifier, + statementType: String): Unit = { + val methods = GeneratedColumn.getClass.getMethods + val (method, args) = methods.find(_.getName == "validateGeneratedColumns") match { + case Some(m) => (m, Seq(tableSchema, catalog, ident, statementType)) + case None => + val m = methods + .find(_.getName == "validateCatalogForGeneratedColumn") + .getOrElse(throw new IllegalStateException("No generated column validation in Spark")) + (m, Seq(columns, catalog, ident)) + } + try { method.invoke(GeneratedColumn, args: _*) } + catch { case e: java.lang.reflect.InvocationTargetException => throw e.getCause } + } + private def invalidateCache(tableCatalog: TableCatalog, ident: Identifier): Unit = { tableCatalog.invalidateTable(ident) } @@ -422,7 +443,7 @@ class Spark4Shim extends SparkShim { readSpatial(array, pos, geography = true) private def readSpatial(value: AnyRef, pos: Int, geography: Boolean): Array[Byte] = { - val modern = org.apache.spark.SPARK_VERSION.startsWith("4.2.") + val modern = org.apache.spark.SPARK_VERSION >= "4.2" val getter = if (modern) "getBinaryView" else if (geography) "getGeography" else "getGeometry" val raw = value.getClass .getMethod(getter, java.lang.Integer.TYPE) @@ -447,7 +468,7 @@ class Spark4Shim extends SparkShim { override def toSparkGeography(wkb: Array[Byte], crs: String, algorithm: String): Object = { val geographyType = sparkGeographyType(crs, algorithm) val name = - if (org.apache.spark.SPARK_VERSION.startsWith("4.2.")) "stGeogSetSrid" else "stSetSrid" + if (org.apache.spark.SPARK_VERSION >= "4.2") "stGeogSetSrid" else "stSetSrid" val raw = STUtils.stGeogFromWKB(wkb) val method = Class .forName("org.apache.spark.sql.catalyst.util.STUtils") diff --git a/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/catalyst/parser/extensions/Spark43SqlExtensionsParser.scala b/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/catalyst/parser/extensions/Spark43SqlExtensionsParser.scala new file mode 100644 index 000000000000..fb3fa9116896 --- /dev/null +++ b/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/catalyst/parser/extensions/Spark43SqlExtensionsParser.scala @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.catalyst.parser.extensions + +import org.apache.spark.sql.catalyst.parser.{ParserInterface, SqlStatementSplitResult} +import org.apache.spark.sql.catalyst.parser.extensions.AbstractPaimonSparkSqlExtensionsParser +import org.apache.spark.sql.types.StructType + +/** Spark 4.3 adds `splitStatements` to `ParserInterface`. */ +class PaimonSpark4SqlExtensionsParser(override val delegate: ParserInterface) + extends AbstractPaimonSparkSqlExtensionsParser(delegate) + with ParserInterface { + + override def parseRoutineParam(sqlText: String): StructType = delegate.parseRoutineParam(sqlText) + + override def splitStatements(sqlText: String): SqlStatementSplitResult = + delegate.splitStatements(sqlText) +} diff --git a/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/data/Spark43ArrayData.scala b/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/data/Spark43ArrayData.scala new file mode 100644 index 000000000000..d06f843801ba --- /dev/null +++ b/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/data/Spark43ArrayData.scala @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.data + +import org.apache.paimon.types.{DataType, GeographyType, GeometryType} + +import org.apache.spark.sql.paimon.shims.SparkShimLoader +import org.apache.spark.unsafe.types.{BinaryView, TimestampNanosVal, VariantVal} + +/** Spark 4.3 array getters: BinaryView spatial values, as on 4.2, plus nanosecond timestamps. */ +class Spark4ArrayData(override val elementType: DataType) extends AbstractSparkArrayData { + override def getVariant(ordinal: Int): VariantVal = { + val value = paimonArray.getVariant(ordinal) + new VariantVal(value.value(), value.metadata()) + } + + override def getBinaryView(ordinal: Int): BinaryView = { + val bytes = paimonArray.getBinary(ordinal) + val value = elementType match { + case geometry: GeometryType => + SparkShimLoader.shim.toSparkGeometry(bytes, geometry.getCrs) + case geography: GeographyType => + SparkShimLoader.shim.toSparkGeography( + bytes, + geography.getCrs, + geography.getAlgorithm.toString) + case other => throw new UnsupportedOperationException(s"No BinaryView for $other") + } + value.asInstanceOf[BinaryView] + } + + override def getTimestampNTZNanos(ordinal: Int): TimestampNanosVal = + throw new UnsupportedOperationException("No Paimon type maps to Spark nanosecond timestamps") + + override def getTimestampLTZNanos(ordinal: Int): TimestampNanosVal = + throw new UnsupportedOperationException("No Paimon type maps to Spark nanosecond timestamps") +} diff --git a/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/data/Spark43InternalRow.scala b/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/data/Spark43InternalRow.scala new file mode 100644 index 000000000000..398c4caa46be --- /dev/null +++ b/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/data/Spark43InternalRow.scala @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.data + +import org.apache.paimon.spark.AbstractSparkInternalRow +import org.apache.paimon.types.{GeographyType, GeometryType, RowType} + +import org.apache.spark.sql.paimon.shims.SparkShimLoader +import org.apache.spark.unsafe.types.{BinaryView, TimestampNanosVal, VariantVal} + +/** Spark 4.3 row getters: BinaryView spatial values, as on 4.2, plus nanosecond timestamps. */ +class Spark4InternalRow(rowType: RowType) extends AbstractSparkInternalRow(rowType) { + override def getVariant(i: Int): VariantVal = { + val value = row.getVariant(i) + new VariantVal(value.value(), value.metadata()) + } + + override def getBinaryView(ordinal: Int): BinaryView = { + val bytes = row.getBinary(ordinal) + val value = rowType.getTypeAt(ordinal) match { + case geometry: GeometryType => + SparkShimLoader.shim.toSparkGeometry(bytes, geometry.getCrs) + case geography: GeographyType => + SparkShimLoader.shim.toSparkGeography( + bytes, + geography.getCrs, + geography.getAlgorithm.toString) + case other => throw new UnsupportedOperationException(s"No BinaryView for $other") + } + value.asInstanceOf[BinaryView] + } + + override def getTimestampNTZNanos(ordinal: Int): TimestampNanosVal = + throw new UnsupportedOperationException("No Paimon type maps to Spark nanosecond timestamps") + + override def getTimestampLTZNanos(ordinal: Int): TimestampNanosVal = + throw new UnsupportedOperationException("No Paimon type maps to Spark nanosecond timestamps") +} diff --git a/pom.xml b/pom.xml index 2fc24798b0e0..598350fa831e 100644 --- a/pom.xml +++ b/pom.xml @@ -475,6 +475,25 @@ under the License. + + spark43 + + paimon-spark/paimon-spark4-common + paimon-spark/paimon-spark-4.3 + + + 17 + 4.13.1 + 2.13 + 2.13.18 + 4.3.0 + paimon-spark4-common_2.13 + 18.1.0 + 4.3 + 4.3.0 + + + flink1 diff --git a/tools/ci/plan.py b/tools/ci/plan.py index e0433ee52e0a..6a01cee53bf1 100644 --- a/tools/ci/plan.py +++ b/tools/ci/plan.py @@ -35,6 +35,7 @@ ("spark3", "Spark 3 / Scala 2.13", 8, "2.13", 90), ("spark4", "Spark 4 / Scala 2.13", 17, "2.13", 90), ("spark42", "Spark 4.2 / Scala 2.13", 17, "2.13", 90), + ("spark43", "Spark 4.3 / Scala 2.13", 17, "2.13", 90), ("e2e-flink1", "E2E / Flink 1 and Spark interoperability", 8, "2.12", 60), ("e2e-flink2", "E2E / Flink 2", 11, "2.12", 60), ("iceberg", "Iceberg GA / JDK 17", 17, "2.12", 60), @@ -46,7 +47,7 @@ REST_SPECS = {"docs/static/rest-catalog-open-api.yaml", "docs/static/rest-management-open-api.yaml"} OPENAPI_V4_ACTIONS = "paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java" FLINK = {"flink1-common", "flink1-connectors", "flink2"} -SPARK = {"spark3", "spark4", "spark42"} +SPARK = {"spark3", "spark4", "spark42", "spark43"} E2E = {"e2e-flink1", "e2e-flink2"} diff --git a/tools/ci/run-java-tests.sh b/tools/ci/run-java-tests.sh index eb8061977fe4..30cea6c63876 100644 --- a/tools/ci/run-java-tests.sh +++ b/tools/ci/run-java-tests.sh @@ -69,6 +69,10 @@ case "$suite" in profiles=flink1,spark42 modules=(org.apache.paimon:paimon-spark-4.2_2.13) ;; + spark43) + profiles=flink1,spark43 + modules=(org.apache.paimon:paimon-spark-4.3_2.13) + ;; e2e-flink1) profiles=flink1,spark3,flink-1.20 modules=(paimon-e2e-tests) diff --git a/tools/ci/test_java_commands.py b/tools/ci/test_java_commands.py index 346ceccc4fef..05d887264a59 100644 --- a/tools/ci/test_java_commands.py +++ b/tools/ci/test_java_commands.py @@ -66,6 +66,9 @@ def test_spark_retains_all_connector_versions_and_scala_variants(self): spark42 = self.commands('spark42', '2.13', 17)[1] self.assertEqual(self.modules(spark42), {'org.apache.paimon:paimon-spark-4.2_2.13'}) self.assertIn('-Pflink1,spark42', spark42) + spark43 = self.commands('spark43', '2.13', 17)[1] + self.assertEqual(self.modules(spark43), {'org.apache.paimon:paimon-spark-4.3_2.13'}) + self.assertIn('-Pflink1,spark43', spark43) def test_core_keeps_common_and_docs_tests_and_existing_jdk11_exclusion(self): for java in (8, 11): diff --git a/tools/ci/test_plan.py b/tools/ci/test_plan.py index 89bcf71c6e0f..58990f360c12 100644 --- a/tools/ci/test_plan.py +++ b/tools/ci/test_plan.py @@ -35,7 +35,7 @@ def suites(self, paths): def test_spark_keeps_common_docs_and_interoperability_tests(self): suites = self.suites(['paimon-spark/paimon-spark-common/src/main/scala/Change.scala']) - self.assertEqual(suites, {'core', 'spark3', 'spark4', 'spark42', 'e2e-flink1'}) + self.assertEqual(suites, {'core', 'spark3', 'spark4', 'spark42', 'spark43', 'e2e-flink1'}) def test_flink_keeps_hive_iceberg_and_both_e2e_versions(self): suites = self.suites(['paimon-flink/paimon-flink-common/src/main/java/Change.java']) @@ -64,7 +64,7 @@ def test_docs_and_markdown_do_not_start_maven(self): def test_mixed_changes_take_union(self): self.assertEqual(self.suites(['docs/intro.md', 'paimon-spark/pom.xml', 'paimon-flink/pom.xml']), - {'core', 'spark3', 'spark4', 'spark42', 'flink1-common', + {'core', 'spark3', 'spark4', 'spark42', 'spark43', 'flink1-common', 'flink1-connectors', 'flink2', 'e2e-flink1', 'e2e-flink2', 'iceberg'}) result = plan.select(['paimon-python/setup.py', 'paimon-core/Change.java']) @@ -220,7 +220,7 @@ def test_pr_size_guard_ignores_unchanged_but_rejects_changed_oversized_files(sel base = self.git('rev-parse', 'HEAD') self.commit('paimon-spark/Change.scala', 'spark') outputs, _ = self.run_pr_plan(base, self.git('rev-parse', 'HEAD')) - self.assertEqual(len(outputs['matrix']['include']), 7) + self.assertEqual(len(outputs['matrix']['include']), 8) for path in ('docs/static/old image.png', 'docs/static/new image.png'): with self.subTest(path=path): @@ -241,7 +241,7 @@ def test_github_outputs_are_valid_json_and_summary_explains_selection(self): outputs = dict(line.split('=', 1) for line in Path('output.txt').read_text().splitlines()) self.assertTrue(json.loads(outputs['java'])) self.assertFalse(json.loads(outputs['python'])) - self.assertEqual(len(json.loads(outputs['matrix'])['include']), 7) + self.assertEqual(len(json.loads(outputs['matrix'])['include']), 8) self.assertIn('Spark interoperability', Path('summary.md').read_text())