From d28e7c20b5fef459b36c16bfd9bd7d26a4ab166d Mon Sep 17 00:00:00 2001 From: QuakeWang Date: Fri, 20 Nov 2026 14:49:45 +0800 Subject: [PATCH] [spark] Support EXTRACT in Spark 4.2 Ossie semantic views The Ossie expression allowlist rejects EXTRACT during query analysis. Map YEAR, MONTH, and DAY in the standard FROM form to Spark builtins so Ossie models can group measures by integer date parts. Signed-off-by: QuakeWang --- docs/docs/concepts/rest/semantic-views.md | 5 +- .../sql/paimon/OssieToSparkMetricView.scala | 38 +++++-- .../sql/paimon/OssieSemanticViewTest.scala | 100 +++++++++++++++++- 3 files changed, 134 insertions(+), 9 deletions(-) diff --git a/docs/docs/concepts/rest/semantic-views.md b/docs/docs/concepts/rest/semantic-views.md index 23af8dc2c9bd..36660b48b118 100644 --- a/docs/docs/concepts/rest/semantic-views.md +++ b/docs/docs/concepts/rest/semantic-views.md @@ -223,13 +223,16 @@ This experimental execution profile supports: - Portable `expression.sql` or an `OSSIE_SQL_2026` dialect expression; arithmetic, comparisons, boolean logic, `CASE`, `IN`, numeric/string/boolean/date casts, `SUM`, `COUNT` (including distinct), `AVG`, `MIN`, `MAX`, `ABS`, `ROUND`, `FLOOR`, `CEIL`, `CEILING`, `COALESCE`, - and `DATE_TRUNC`. + `DATE_TRUNC`, and `EXTRACT`. - `DATE_TRUNC(unit, input)` requires a case-insensitive string literal for the unit. The supported units are `year`, `quarter`, `month`, `week`, `day`, `hour`, `minute`, and `second`. Weeks start on Monday. The result is `TIMESTAMP_LTZ`. Spark uses the session time zone to truncate `TIMESTAMP_LTZ` values and implicitly convert `DATE` values and timestamp strings to `TIMESTAMP_LTZ`. A `NULL` input returns `NULL`. `TIMESTAMP_NTZ` inputs are rejected during analysis. +- `EXTRACT(unit FROM date_expr)` supports `YEAR`, `MONTH`, and `DAY` (day of month), + case-insensitively. It returns an integer and propagates `NULL`. Spark applies its + built-in date input coercion. Other expressions, relationships, multiple datasets, external joins, streaming reads and writes are rejected. SQL Semantic View DDL and SQL discovery are not added; use the management API to diff --git a/paimon-spark/paimon-spark-4.2/src/main/scala/org/apache/spark/sql/paimon/OssieToSparkMetricView.scala b/paimon-spark/paimon-spark-4.2/src/main/scala/org/apache/spark/sql/paimon/OssieToSparkMetricView.scala index 441f785915ec..3cbcd100cb83 100644 --- a/paimon-spark/paimon-spark-4.2/src/main/scala/org/apache/spark/sql/paimon/OssieToSparkMetricView.scala +++ b/paimon-spark/paimon-spark-4.2/src/main/scala/org/apache/spark/sql/paimon/OssieToSparkMetricView.scala @@ -41,10 +41,19 @@ import scala.collection.mutable private[paimon] object OssieToSparkMetricView { private val aggregates = Set("sum", "count", "avg", "min", "max") private val functions = - aggregates ++ Set("abs", "round", "floor", "ceil", "ceiling", "coalesce", "date_trunc") + aggregates ++ Set( + "abs", + "round", + "floor", + "ceil", + "ceiling", + "coalesce", + "date_trunc", + "extract") private val dateTruncUnits = Set("YEAR", "QUARTER", "MONTH", "WEEK", "DAY", "HOUR", "MINUTE", "SECOND") private val dateTruncFunction = Seq("system", "builtin", "date_trunc") + private val extractFunctions = Map("YEAR" -> "year", "MONTH" -> "month", "DAY" -> "dayofmonth") case class Conversion(yaml: String, dateTruncOrigins: Set[Origin]) @@ -161,7 +170,7 @@ private[paimon] object OssieToSparkMetricView { val expr = SQLConf.withExistingConf(conf)(CatalystSqlParser.parseExpression(normalizeStrings(sql))) var aggregate = false - expr.foreach { + val validated = expr.transformDown { case f: UnresolvedFunction => require( f.ignoreNulls.isEmpty && f.orderingWithinGroup.isEmpty, @@ -189,22 +198,39 @@ private[paimon] object OssieToSparkMetricView { require( !f.arguments.exists(_.isInstanceOf[UnresolvedStar]) || name == "count", "Only COUNT(*) supports a star in Ossie expressions") - case _: Literal | _: UnresolvedAttribute | _: UnresolvedStar => - case _: Add | _: Subtract | _: Multiply | _: Divide | _: Remainder | _: UnaryMinus | + if (name == "extract") { + f.arguments match { + // Spark constructs the unit literal at the call's origin only for EXTRACT(... FROM ...). + case Seq(unit @ Literal(value: UTF8String, _: StringType), input) + if unit.origin == f.origin => + val builtin = extractFunctions.getOrElse( + value.toString.toUpperCase(Locale.ROOT), + throw new IllegalArgumentException(s"Unsupported Ossie EXTRACT unit: $value")) + f.copy(nameParts = Seq("system", "builtin", builtin), arguments = Seq(input)) + case _ => + throw new IllegalArgumentException("Ossie EXTRACT expects EXTRACT(unit FROM input)") + } + } else { + f + } + case e @ (_: Literal | _: UnresolvedAttribute | _: UnresolvedStar) => e + case e @ (_: Add | _: Subtract | _: Multiply | _: Divide | _: Remainder | _: UnaryMinus | _: UnaryPositive | _: CaseWhen | _: EqualTo | _: LessThan | _: LessThanOrEqual | _: GreaterThan | _: GreaterThanOrEqual | _: And | _: Or | _: Not | _: IsNull | - _: IsNotNull | _: In => + _: IsNotNull | _: In) => + e case c: Cast => require( c.dataType.isInstanceOf[NumericType] || c.dataType == StringType || c.dataType == BooleanType || c.dataType == DateType, s"Unsupported Ossie cast type: ${c.dataType}" ) + c case other => throw new IllegalArgumentException(s"Unsupported OSSIE_SQL_2026 expression: ${other.sql}") } require(!metric || aggregate, "Ossie metrics must contain a supported aggregate") - expr + validated } // Ossie strings use ANSI doubled quotes and literal backslashes. Normalize only string tokens diff --git a/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala index f783e4a47014..5e1d4b5368f7 100644 --- a/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala +++ b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala @@ -30,9 +30,9 @@ import org.apache.paimon.view.SemanticViewDefinition import okhttp3.mockwebserver.{Dispatcher, MockResponse, RecordedRequest} import org.apache.spark.SparkConf -import org.apache.spark.sql.{QueryTest, Row} +import org.apache.spark.sql.{AnalysisException, QueryTest, Row} import org.apache.spark.sql.test.SharedSparkSession -import org.apache.spark.sql.types.TimestampType +import org.apache.spark.sql.types.{IntegerType, TimestampType} import java.nio.file.Files import java.time.Instant @@ -119,6 +119,102 @@ class OssieSemanticViewTest extends QueryTest with SharedSparkSession { Seq(Row(null, 0L))) } + test("group Ossie measures by integer EXTRACT dimensions") { + spark.sql( + "CREATE TABLE paimon.sales.extract_orders (order_date DATE, amount BIGINT) " + + "TBLPROPERTIES ('bucket' = '-1', 'file.format' = 'avro')") + spark.sql("""INSERT INTO paimon.sales.extract_orders VALUES + | (DATE '2025-05-10', 5), + | (DATE '2026-05-10', 10), + | (DATE '2026-05-31', 20), + | (DATE '2026-06-01', 7), + | (NULL, 3)""".stripMargin) + register("""version: 0.3.0.dev0 + |name: order_metrics + |datasets: + | - name: orders + | source: extract_orders + | fields: + | - name: order_year + | expression: + | sql: ExTrAcT(YeAr FROM "ORDER_DATE") + | - name: order_month + | expression: + | sql: EXTRACT(MONTH FROM COALESCE(order_date, NULL)) + | - name: order_day + | expression: + | sql: EXTRACT(DAY FROM order_date) + | metrics: + | - name: revenue + | expression: + | sql: SUM(source.amount) + | - name: latest_order_year + | expression: + | sql: MAX(EXTRACT(YEAR FROM source.order_date)) + |""".stripMargin) + + val expected = Seq( + "order_year" -> Seq(Row(2025, 5L), Row(2026, 37L), Row(null, 3L)), + "order_month" -> Seq(Row(5, 35L), Row(6, 7L), Row(null, 3L)), + "order_day" -> Seq(Row(1, 7L), Row(10, 15L), Row(31, 20L), Row(null, 3L)) + ) + expected.foreach { + case (dimension, rows) => + val result = spark.sql( + s"SELECT $dimension, MEASURE(revenue) AS revenue " + + s"FROM paimon.sales.order_metrics GROUP BY $dimension") + assert(result.schema(dimension).dataType == IntegerType) + checkAnswer(result, rows) + } + checkAnswer( + spark.sql("SELECT MEASURE(latest_order_year) FROM paimon.sales.order_metrics"), + Row(2026)) + } + + test("reject unsupported Ossie date extraction syntax and inputs") { + val date = "DATE '2026-06-01'" + val query = "SELECT region FROM paimon.sales.order_metrics" + Seq( + s"EXTRACT(QUARTER FROM $date)" -> "Unsupported Ossie EXTRACT unit: QUARTER", + s"EXTRACT(Y FROM $date)" -> "Unsupported Ossie EXTRACT unit: Y", + s"extract('year', $date)" -> "Ossie EXTRACT expects EXTRACT(unit FROM input)", + s"DATE_PART('year', $date)" -> "Unsupported OSSIE_SQL_2026 function: DATE_PART", + s"YEAR($date)" -> "Unsupported OSSIE_SQL_2026 function: YEAR" + ).foreach { + case (expression, message) => + register(model.replace("sql: region_code", s"sql: $expression")) + val error = intercept[IllegalArgumentException] { + spark.sql(query).schema + } + assert(error.getMessage.contains(message), error.toString) + } + + register(model.replace("sql: region_code", "sql: EXTRACT(DAY FROM INTERVAL '2' DAY)")) + val error = intercept[AnalysisException] { + spark.sql(query).schema + } + assert(error.getCondition == "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE", error.toString) + } + + test("resolve Ossie EXTRACT to the Spark builtin regardless of the SQL path") { + val session = spark.newSession() + session.conf.set("spark.sql.path.enabled", "true") + session.sql("CREATE FUNCTION spark_catalog.default.year(d DATE) RETURNS INT RETURN -1") + try { + session.sql("SET PATH = spark_catalog.default, system.builtin") + val date = "DATE '2026-06-01'" + checkAnswer(session.sql(s"SELECT year($date)"), Row(-1)) + + register(model.replace("sql: region_code", s"sql: EXTRACT(YEAR FROM $date)")) + checkAnswer( + session.sql( + "SELECT region, MEASURE(gross) FROM paimon.sales.order_metrics GROUP BY region"), + Row(2026, BigDecimal(280))) + } finally { + session.sql("DROP FUNCTION spark_catalog.default.year") + } + } + test("group Ossie measures by DATE_TRUNC and preserve Spark timestamp semantics") { withSQLConf("spark.sql.session.timeZone" -> "UTC") { spark.sql(