diff --git a/docs/docs/concepts/rest/semantic-views.md b/docs/docs/concepts/rest/semantic-views.md index 23af8dc2c9bd..36660b48b118 100644 --- a/docs/docs/concepts/rest/semantic-views.md +++ b/docs/docs/concepts/rest/semantic-views.md @@ -223,13 +223,16 @@ This experimental execution profile supports: - Portable `expression.sql` or an `OSSIE_SQL_2026` dialect expression; arithmetic, comparisons, boolean logic, `CASE`, `IN`, numeric/string/boolean/date casts, `SUM`, `COUNT` (including distinct), `AVG`, `MIN`, `MAX`, `ABS`, `ROUND`, `FLOOR`, `CEIL`, `CEILING`, `COALESCE`, - and `DATE_TRUNC`. + `DATE_TRUNC`, and `EXTRACT`. - `DATE_TRUNC(unit, input)` requires a case-insensitive string literal for the unit. The supported units are `year`, `quarter`, `month`, `week`, `day`, `hour`, `minute`, and `second`. Weeks start on Monday. The result is `TIMESTAMP_LTZ`. Spark uses the session time zone to truncate `TIMESTAMP_LTZ` values and implicitly convert `DATE` values and timestamp strings to `TIMESTAMP_LTZ`. A `NULL` input returns `NULL`. `TIMESTAMP_NTZ` inputs are rejected during analysis. +- `EXTRACT(unit FROM date_expr)` supports `YEAR`, `MONTH`, and `DAY` (day of month), + case-insensitively. It returns an integer and propagates `NULL`. Spark applies its + built-in date input coercion. Other expressions, relationships, multiple datasets, external joins, streaming reads and writes are rejected. SQL Semantic View DDL and SQL discovery are not added; use the management API to diff --git a/paimon-spark/paimon-spark-4.2/src/main/scala/org/apache/spark/sql/paimon/OssieToSparkMetricView.scala b/paimon-spark/paimon-spark-4.2/src/main/scala/org/apache/spark/sql/paimon/OssieToSparkMetricView.scala index 441f785915ec..3cbcd100cb83 100644 --- a/paimon-spark/paimon-spark-4.2/src/main/scala/org/apache/spark/sql/paimon/OssieToSparkMetricView.scala +++ b/paimon-spark/paimon-spark-4.2/src/main/scala/org/apache/spark/sql/paimon/OssieToSparkMetricView.scala @@ -41,10 +41,19 @@ import scala.collection.mutable private[paimon] object OssieToSparkMetricView { private val aggregates = Set("sum", "count", "avg", "min", "max") private val functions = - aggregates ++ Set("abs", "round", "floor", "ceil", "ceiling", "coalesce", "date_trunc") + aggregates ++ Set( + "abs", + "round", + "floor", + "ceil", + "ceiling", + "coalesce", + "date_trunc", + "extract") private val dateTruncUnits = Set("YEAR", "QUARTER", "MONTH", "WEEK", "DAY", "HOUR", "MINUTE", "SECOND") private val dateTruncFunction = Seq("system", "builtin", "date_trunc") + private val extractFunctions = Map("YEAR" -> "year", "MONTH" -> "month", "DAY" -> "dayofmonth") case class Conversion(yaml: String, dateTruncOrigins: Set[Origin]) @@ -161,7 +170,7 @@ private[paimon] object OssieToSparkMetricView { val expr = SQLConf.withExistingConf(conf)(CatalystSqlParser.parseExpression(normalizeStrings(sql))) var aggregate = false - expr.foreach { + val validated = expr.transformDown { case f: UnresolvedFunction => require( f.ignoreNulls.isEmpty && f.orderingWithinGroup.isEmpty, @@ -189,22 +198,39 @@ private[paimon] object OssieToSparkMetricView { require( !f.arguments.exists(_.isInstanceOf[UnresolvedStar]) || name == "count", "Only COUNT(*) supports a star in Ossie expressions") - case _: Literal | _: UnresolvedAttribute | _: UnresolvedStar => - case _: Add | _: Subtract | _: Multiply | _: Divide | _: Remainder | _: UnaryMinus | + if (name == "extract") { + f.arguments match { + // Spark constructs the unit literal at the call's origin only for EXTRACT(... FROM ...). + case Seq(unit @ Literal(value: UTF8String, _: StringType), input) + if unit.origin == f.origin => + val builtin = extractFunctions.getOrElse( + value.toString.toUpperCase(Locale.ROOT), + throw new IllegalArgumentException(s"Unsupported Ossie EXTRACT unit: $value")) + f.copy(nameParts = Seq("system", "builtin", builtin), arguments = Seq(input)) + case _ => + throw new IllegalArgumentException("Ossie EXTRACT expects EXTRACT(unit FROM input)") + } + } else { + f + } + case e @ (_: Literal | _: UnresolvedAttribute | _: UnresolvedStar) => e + case e @ (_: Add | _: Subtract | _: Multiply | _: Divide | _: Remainder | _: UnaryMinus | _: UnaryPositive | _: CaseWhen | _: EqualTo | _: LessThan | _: LessThanOrEqual | _: GreaterThan | _: GreaterThanOrEqual | _: And | _: Or | _: Not | _: IsNull | - _: IsNotNull | _: In => + _: IsNotNull | _: In) => + e case c: Cast => require( c.dataType.isInstanceOf[NumericType] || c.dataType == StringType || c.dataType == BooleanType || c.dataType == DateType, s"Unsupported Ossie cast type: ${c.dataType}" ) + c case other => throw new IllegalArgumentException(s"Unsupported OSSIE_SQL_2026 expression: ${other.sql}") } require(!metric || aggregate, "Ossie metrics must contain a supported aggregate") - expr + validated } // Ossie strings use ANSI doubled quotes and literal backslashes. Normalize only string tokens diff --git a/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala index f783e4a47014..5e1d4b5368f7 100644 --- a/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala +++ b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala @@ -30,9 +30,9 @@ import org.apache.paimon.view.SemanticViewDefinition import okhttp3.mockwebserver.{Dispatcher, MockResponse, RecordedRequest} import org.apache.spark.SparkConf -import org.apache.spark.sql.{QueryTest, Row} +import org.apache.spark.sql.{AnalysisException, QueryTest, Row} import org.apache.spark.sql.test.SharedSparkSession -import org.apache.spark.sql.types.TimestampType +import org.apache.spark.sql.types.{IntegerType, TimestampType} import java.nio.file.Files import java.time.Instant @@ -119,6 +119,102 @@ class OssieSemanticViewTest extends QueryTest with SharedSparkSession { Seq(Row(null, 0L))) } + test("group Ossie measures by integer EXTRACT dimensions") { + spark.sql( + "CREATE TABLE paimon.sales.extract_orders (order_date DATE, amount BIGINT) " + + "TBLPROPERTIES ('bucket' = '-1', 'file.format' = 'avro')") + spark.sql("""INSERT INTO paimon.sales.extract_orders VALUES + | (DATE '2025-05-10', 5), + | (DATE '2026-05-10', 10), + | (DATE '2026-05-31', 20), + | (DATE '2026-06-01', 7), + | (NULL, 3)""".stripMargin) + register("""version: 0.3.0.dev0 + |name: order_metrics + |datasets: + | - name: orders + | source: extract_orders + | fields: + | - name: order_year + | expression: + | sql: ExTrAcT(YeAr FROM "ORDER_DATE") + | - name: order_month + | expression: + | sql: EXTRACT(MONTH FROM COALESCE(order_date, NULL)) + | - name: order_day + | expression: + | sql: EXTRACT(DAY FROM order_date) + | metrics: + | - name: revenue + | expression: + | sql: SUM(source.amount) + | - name: latest_order_year + | expression: + | sql: MAX(EXTRACT(YEAR FROM source.order_date)) + |""".stripMargin) + + val expected = Seq( + "order_year" -> Seq(Row(2025, 5L), Row(2026, 37L), Row(null, 3L)), + "order_month" -> Seq(Row(5, 35L), Row(6, 7L), Row(null, 3L)), + "order_day" -> Seq(Row(1, 7L), Row(10, 15L), Row(31, 20L), Row(null, 3L)) + ) + expected.foreach { + case (dimension, rows) => + val result = spark.sql( + s"SELECT $dimension, MEASURE(revenue) AS revenue " + + s"FROM paimon.sales.order_metrics GROUP BY $dimension") + assert(result.schema(dimension).dataType == IntegerType) + checkAnswer(result, rows) + } + checkAnswer( + spark.sql("SELECT MEASURE(latest_order_year) FROM paimon.sales.order_metrics"), + Row(2026)) + } + + test("reject unsupported Ossie date extraction syntax and inputs") { + val date = "DATE '2026-06-01'" + val query = "SELECT region FROM paimon.sales.order_metrics" + Seq( + s"EXTRACT(QUARTER FROM $date)" -> "Unsupported Ossie EXTRACT unit: QUARTER", + s"EXTRACT(Y FROM $date)" -> "Unsupported Ossie EXTRACT unit: Y", + s"extract('year', $date)" -> "Ossie EXTRACT expects EXTRACT(unit FROM input)", + s"DATE_PART('year', $date)" -> "Unsupported OSSIE_SQL_2026 function: DATE_PART", + s"YEAR($date)" -> "Unsupported OSSIE_SQL_2026 function: YEAR" + ).foreach { + case (expression, message) => + register(model.replace("sql: region_code", s"sql: $expression")) + val error = intercept[IllegalArgumentException] { + spark.sql(query).schema + } + assert(error.getMessage.contains(message), error.toString) + } + + register(model.replace("sql: region_code", "sql: EXTRACT(DAY FROM INTERVAL '2' DAY)")) + val error = intercept[AnalysisException] { + spark.sql(query).schema + } + assert(error.getCondition == "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE", error.toString) + } + + test("resolve Ossie EXTRACT to the Spark builtin regardless of the SQL path") { + val session = spark.newSession() + session.conf.set("spark.sql.path.enabled", "true") + session.sql("CREATE FUNCTION spark_catalog.default.year(d DATE) RETURNS INT RETURN -1") + try { + session.sql("SET PATH = spark_catalog.default, system.builtin") + val date = "DATE '2026-06-01'" + checkAnswer(session.sql(s"SELECT year($date)"), Row(-1)) + + register(model.replace("sql: region_code", s"sql: EXTRACT(YEAR FROM $date)")) + checkAnswer( + session.sql( + "SELECT region, MEASURE(gross) FROM paimon.sales.order_metrics GROUP BY region"), + Row(2026, BigDecimal(280))) + } finally { + session.sql("DROP FUNCTION spark_catalog.default.year") + } + } + test("group Ossie measures by DATE_TRUNC and preserve Spark timestamp semantics") { withSQLConf("spark.sql.session.timeZone" -> "UTC") { spark.sql(