Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion docs/docs/concepts/rest/semantic-views.md
Original file line number Diff line number Diff line change
Expand Up @@ -223,13 +223,16 @@ This experimental execution profile supports:
- Portable `expression.sql` or an `OSSIE_SQL_2026` dialect expression; arithmetic, comparisons,
boolean logic, `CASE`, `IN`, numeric/string/boolean/date casts, `SUM`, `COUNT` (including
distinct), `AVG`, `MIN`, `MAX`, `ABS`, `ROUND`, `FLOOR`, `CEIL`, `CEILING`, `COALESCE`,
and `DATE_TRUNC`.
`DATE_TRUNC`, and `EXTRACT`.
- `DATE_TRUNC(unit, input)` requires a case-insensitive string literal for the unit. The supported
units are `year`, `quarter`, `month`, `week`, `day`, `hour`, `minute`, and `second`. Weeks
start on Monday. The result is `TIMESTAMP_LTZ`. Spark uses the session time zone to truncate
`TIMESTAMP_LTZ` values and implicitly convert `DATE` values and timestamp strings to
`TIMESTAMP_LTZ`. A `NULL` input returns `NULL`. `TIMESTAMP_NTZ` inputs are rejected during
analysis.
- `EXTRACT(unit FROM date_expr)` supports `YEAR`, `MONTH`, and `DAY` (day of month),
case-insensitively. It returns an integer and propagates `NULL`. Spark applies its
built-in date input coercion.

Other expressions, relationships, multiple datasets, external joins, streaming reads and writes
are rejected. SQL Semantic View DDL and SQL discovery are not added; use the management API to
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,19 @@ import scala.collection.mutable
private[paimon] object OssieToSparkMetricView {
private val aggregates = Set("sum", "count", "avg", "min", "max")
private val functions =
aggregates ++ Set("abs", "round", "floor", "ceil", "ceiling", "coalesce", "date_trunc")
aggregates ++ Set(
"abs",
"round",
"floor",
"ceil",
"ceiling",
"coalesce",
"date_trunc",
"extract")
private val dateTruncUnits =
Set("YEAR", "QUARTER", "MONTH", "WEEK", "DAY", "HOUR", "MINUTE", "SECOND")
private val dateTruncFunction = Seq("system", "builtin", "date_trunc")
private val extractFunctions = Map("YEAR" -> "year", "MONTH" -> "month", "DAY" -> "dayofmonth")

case class Conversion(yaml: String, dateTruncOrigins: Set[Origin])

Expand Down Expand Up @@ -161,7 +170,7 @@ private[paimon] object OssieToSparkMetricView {
val expr =
SQLConf.withExistingConf(conf)(CatalystSqlParser.parseExpression(normalizeStrings(sql)))
var aggregate = false
expr.foreach {
val validated = expr.transformDown {
case f: UnresolvedFunction =>
require(
f.ignoreNulls.isEmpty && f.orderingWithinGroup.isEmpty,
Expand Down Expand Up @@ -189,22 +198,39 @@ private[paimon] object OssieToSparkMetricView {
require(
!f.arguments.exists(_.isInstanceOf[UnresolvedStar]) || name == "count",
"Only COUNT(*) supports a star in Ossie expressions")
case _: Literal | _: UnresolvedAttribute | _: UnresolvedStar =>
case _: Add | _: Subtract | _: Multiply | _: Divide | _: Remainder | _: UnaryMinus |
if (name == "extract") {
f.arguments match {
// Spark constructs the unit literal at the call's origin only for EXTRACT(... FROM ...).
case Seq(unit @ Literal(value: UTF8String, _: StringType), input)
if unit.origin == f.origin =>
val builtin = extractFunctions.getOrElse(
value.toString.toUpperCase(Locale.ROOT),
throw new IllegalArgumentException(s"Unsupported Ossie EXTRACT unit: $value"))
f.copy(nameParts = Seq("system", "builtin", builtin), arguments = Seq(input))
case _ =>
throw new IllegalArgumentException("Ossie EXTRACT expects EXTRACT(unit FROM input)")
}
} else {
f
}
case e @ (_: Literal | _: UnresolvedAttribute | _: UnresolvedStar) => e
case e @ (_: Add | _: Subtract | _: Multiply | _: Divide | _: Remainder | _: UnaryMinus |
_: UnaryPositive | _: CaseWhen | _: EqualTo | _: LessThan | _: LessThanOrEqual |
_: GreaterThan | _: GreaterThanOrEqual | _: And | _: Or | _: Not | _: IsNull |
_: IsNotNull | _: In =>
_: IsNotNull | _: In) =>
e
case c: Cast =>
require(
c.dataType.isInstanceOf[NumericType] || c.dataType == StringType ||
c.dataType == BooleanType || c.dataType == DateType,
s"Unsupported Ossie cast type: ${c.dataType}"
)
c
case other =>
throw new IllegalArgumentException(s"Unsupported OSSIE_SQL_2026 expression: ${other.sql}")
}
require(!metric || aggregate, "Ossie metrics must contain a supported aggregate")
expr
validated
}

// Ossie strings use ANSI doubled quotes and literal backslashes. Normalize only string tokens
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,9 @@ import org.apache.paimon.view.SemanticViewDefinition

import okhttp3.mockwebserver.{Dispatcher, MockResponse, RecordedRequest}
import org.apache.spark.SparkConf
import org.apache.spark.sql.{QueryTest, Row}
import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.sql.types.TimestampType
import org.apache.spark.sql.types.{IntegerType, TimestampType}

import java.nio.file.Files
import java.time.Instant
Expand Down Expand Up @@ -119,6 +119,102 @@ class OssieSemanticViewTest extends QueryTest with SharedSparkSession {
Seq(Row(null, 0L)))
}

test("group Ossie measures by integer EXTRACT dimensions") {
spark.sql(
"CREATE TABLE paimon.sales.extract_orders (order_date DATE, amount BIGINT) " +
"TBLPROPERTIES ('bucket' = '-1', 'file.format' = 'avro')")
spark.sql("""INSERT INTO paimon.sales.extract_orders VALUES
| (DATE '2025-05-10', 5),
| (DATE '2026-05-10', 10),
| (DATE '2026-05-31', 20),
| (DATE '2026-06-01', 7),
| (NULL, 3)""".stripMargin)
register("""version: 0.3.0.dev0
|name: order_metrics
|datasets:
| - name: orders
| source: extract_orders
| fields:
| - name: order_year
| expression:
| sql: ExTrAcT(YeAr FROM "ORDER_DATE")
| - name: order_month
| expression:
| sql: EXTRACT(MONTH FROM COALESCE(order_date, NULL))
| - name: order_day
| expression:
| sql: EXTRACT(DAY FROM order_date)
| metrics:
| - name: revenue
| expression:
| sql: SUM(source.amount)
| - name: latest_order_year
| expression:
| sql: MAX(EXTRACT(YEAR FROM source.order_date))
|""".stripMargin)

val expected = Seq(
"order_year" -> Seq(Row(2025, 5L), Row(2026, 37L), Row(null, 3L)),
"order_month" -> Seq(Row(5, 35L), Row(6, 7L), Row(null, 3L)),
"order_day" -> Seq(Row(1, 7L), Row(10, 15L), Row(31, 20L), Row(null, 3L))
)
expected.foreach {
case (dimension, rows) =>
val result = spark.sql(
s"SELECT $dimension, MEASURE(revenue) AS revenue " +
s"FROM paimon.sales.order_metrics GROUP BY $dimension")
assert(result.schema(dimension).dataType == IntegerType)
checkAnswer(result, rows)
}
checkAnswer(
spark.sql("SELECT MEASURE(latest_order_year) FROM paimon.sales.order_metrics"),
Row(2026))
}

test("reject unsupported Ossie date extraction syntax and inputs") {
val date = "DATE '2026-06-01'"
val query = "SELECT region FROM paimon.sales.order_metrics"
Seq(
s"EXTRACT(QUARTER FROM $date)" -> "Unsupported Ossie EXTRACT unit: QUARTER",
s"EXTRACT(Y FROM $date)" -> "Unsupported Ossie EXTRACT unit: Y",
s"extract('year', $date)" -> "Ossie EXTRACT expects EXTRACT(unit FROM input)",
s"DATE_PART('year', $date)" -> "Unsupported OSSIE_SQL_2026 function: DATE_PART",
s"YEAR($date)" -> "Unsupported OSSIE_SQL_2026 function: YEAR"
).foreach {
case (expression, message) =>
register(model.replace("sql: region_code", s"sql: $expression"))
val error = intercept[IllegalArgumentException] {
spark.sql(query).schema
}
assert(error.getMessage.contains(message), error.toString)
}

register(model.replace("sql: region_code", "sql: EXTRACT(DAY FROM INTERVAL '2' DAY)"))
val error = intercept[AnalysisException] {
spark.sql(query).schema
}
assert(error.getCondition == "DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE", error.toString)
}

test("resolve Ossie EXTRACT to the Spark builtin regardless of the SQL path") {
val session = spark.newSession()
session.conf.set("spark.sql.path.enabled", "true")
session.sql("CREATE FUNCTION spark_catalog.default.year(d DATE) RETURNS INT RETURN -1")
try {
session.sql("SET PATH = spark_catalog.default, system.builtin")
val date = "DATE '2026-06-01'"
checkAnswer(session.sql(s"SELECT year($date)"), Row(-1))

register(model.replace("sql: region_code", s"sql: EXTRACT(YEAR FROM $date)"))
checkAnswer(
session.sql(
"SELECT region, MEASURE(gross) FROM paimon.sales.order_metrics GROUP BY region"),
Row(2026, BigDecimal(280)))
} finally {
session.sql("DROP FUNCTION spark_catalog.default.year")
}
}

test("group Ossie measures by DATE_TRUNC and preserve Spark timestamp semantics") {
withSQLConf("spark.sql.session.timeZone" -> "UTC") {
spark.sql(
Expand Down
Loading