From d7ac0b38b2850e630bba0966cea248df6472fe83 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Wed, 18 Nov 2026 16:35:04 +0800 Subject: [PATCH 1/3] [spark] Add an isolated Spark 4.3 connector build --- docs/docs/concepts/rest/semantic-views.md | 18 +- docs/docs/spark/installation.mdx | 14 +- paimon-spark/paimon-spark-4.3/pom.xml | 171 ++++++ .../src/main/resources/META-INF/NOTICE | 12 + .../paimon/spark/ossie/ossie-schema.json | 396 ++++++++++++++ .../paimon/spark/ossie/OssieModel.scala | 152 ++++++ .../Spark42RowLevelCommandRewrite.scala | 65 +++ .../sql/paimon/OssieToSparkMetricView.scala | 253 +++++++++ .../paimon/PaimonSemanticViewResolver.scala | 146 +++++ .../data/SparkInternalRowVariantTest.scala | 54 ++ .../paimon/spark/ossie/OssieModelTest.scala | 136 +++++ .../spark/sql/Spark43CompatibilityTest.scala | 230 ++++++++ .../sql/paimon/OssieSemanticViewTest.scala | 497 ++++++++++++++++++ .../PaimonSparkSessionExtensions.scala | 2 +- .../catalog/PaimonV1FunctionRegistry.scala | 4 +- paimon-spark/paimon-spark4-common/pom.xml | 43 ++ .../spark/sql/paimon/shims/Spark4Shim.scala | 27 +- .../Spark43SqlExtensionsParser.scala | 34 ++ .../paimon/spark/data/Spark43ArrayData.scala | 53 ++ .../spark/data/Spark43InternalRow.scala | 54 ++ pom.xml | 19 + tools/ci/plan.py | 3 +- tools/ci/run-java-tests.sh | 4 + tools/ci/test_java_commands.py | 3 + tools/ci/test_plan.py | 8 +- 25 files changed, 2374 insertions(+), 24 deletions(-) create mode 100644 paimon-spark/paimon-spark-4.3/pom.xml create mode 100644 paimon-spark/paimon-spark-4.3/src/main/resources/META-INF/NOTICE create mode 100644 paimon-spark/paimon-spark-4.3/src/main/resources/org/apache/paimon/spark/ossie/ossie-schema.json create mode 100644 paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/paimon/spark/ossie/OssieModel.scala create mode 100644 paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/catalyst/analysis/Spark42RowLevelCommandRewrite.scala create mode 100644 paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/OssieToSparkMetricView.scala create mode 100644 paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/PaimonSemanticViewResolver.scala create mode 100644 paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/data/SparkInternalRowVariantTest.scala create mode 100644 paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/ossie/OssieModelTest.scala create mode 100644 paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/sql/Spark43CompatibilityTest.scala create mode 100644 paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala create mode 100644 paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/catalyst/parser/extensions/Spark43SqlExtensionsParser.scala create mode 100644 paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/data/Spark43ArrayData.scala create mode 100644 paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/data/Spark43InternalRow.scala diff --git a/docs/docs/concepts/rest/semantic-views.md b/docs/docs/concepts/rest/semantic-views.md index 0f5974c390fd..9e983b2fc851 100644 --- a/docs/docs/concepts/rest/semantic-views.md +++ b/docs/docs/concepts/rest/semantic-views.md @@ -28,7 +28,7 @@ preserves the original document text. Servers validate the model formats, syntax and features they support; consumers provide query execution. Registration alone does not grant access to source data or guarantee engine support. -The Spark 4.2 connector supports the [Ossie query profile](#spark-42-ossie-queries) below. +The Spark 4.2 and 4.3 connectors support the [Ossie query profile](#spark-42-ossie-queries) below. This implementation supplies Java clients, wire models, and the [Catalog OpenAPI contract](/rest-catalog-open-api.yaml). REST server providers must implement persistence, validation, authorization, and the atomic operations described below. @@ -168,12 +168,13 @@ Client-side invalid input throws `IllegalArgumentException` before HTTP. An unsu never falls back to SQL views, labels, or table options. Materialization, rename, and per-measure or per-field REST resources are outside this version. -## Spark 4.2 Ossie queries +## Spark 4.2 and 4.3 Ossie queries {#spark-42-ossie-queries} -The `paimon-spark-4.2_2.13` connector can query an `ossie-yaml` definition through the configured -Paimon REST Catalog. Enable `PaimonSparkSessionExtensions` as described in the -[Spark installation guide](../../spark/installation.mdx). The REST server must implement semantic -view management; the connector does not provide production REST persistence or validation. +The `paimon-spark-4.2_2.13` and `paimon-spark-4.3_2.13` connectors can query an `ossie-yaml` +definition through the configured Paimon REST Catalog. Enable `PaimonSparkSessionExtensions` as +described in the [Spark installation guide](../../spark/installation.mdx). The REST server must +implement semantic view management; the connector does not provide production REST persistence or +validation. Register a definition whose content is a flat Ossie document, for example: @@ -230,8 +231,9 @@ register and list definitions. Ordinary tables and SQL views retain their existi Source table authorization is required independently of permission to read the semantic definition. When using a CTE, evaluate measures inside its definition (for example, `WITH m AS (SELECT MEASURE(revenue) AS revenue FROM paimon.sales.order_metrics) SELECT revenue FROM m`). -Forwarding raw measure columns through a CTE or temporary view is not supported by Spark 4.2's -Metric View resolver; evaluate measures inside the defining query block before reusing its results. +Forwarding raw measure columns through a CTE or temporary view is not supported by the Metric View +resolver of Spark 4.2 and 4.3; evaluate measures inside the defining query block before reusing its +results. The bundled schema is pinned to Ossie commit [`2692b1cc4708fdcf6a91e597ca928a9bc9d0da60`](https://github.com/apache/ossie/tree/2692b1cc4708fdcf6a91e597ca928a9bc9d0da60). diff --git a/docs/docs/spark/installation.mdx b/docs/docs/spark/installation.mdx index e223cebe2807..7032587f715b 100644 --- a/docs/docs/spark/installation.mdx +++ b/docs/docs/spark/installation.mdx @@ -34,9 +34,9 @@ may require a newer Java runtime. Check its runtime requirements before starting - Spark 3.x (including 3.5, 3.4, 3.3, 3.2) : Pre-built with Java 8 and Scala 2.12/2.13 -Spark 4.2 is available as an experimental source build with Java 17 and Scala 2.13. -Its independent `spark42` profile recompiles the shared connector modules against Spark 4.2; -the `spark4` profile continues to build the Spark 4.0 and 4.1 connectors. +Spark 4.2 and 4.3 are available as experimental source builds with Java 17 and Scala 2.13. +Their independent `spark42` and `spark43` profiles recompile the shared connector modules against +Spark 4.2 and 4.3; the `spark4` profile continues to build the Spark 4.0 and 4.1 connectors. ## Download the Connector @@ -89,14 +89,18 @@ mvn clean package -DskipTests -pl paimon-spark/paimon-spark-4.1 -am -Pspark4 # build paimon spark 4.2 mvn clean package -DskipTests -pl paimon-spark/paimon-spark-4.2 -am -Pspark42 + +# build paimon spark 4.3 +mvn clean package -DskipTests -pl paimon-spark/paimon-spark-4.3 -am -Pspark43 ``` Use `clean` when switching between Spark profiles: the shared modules produce artifacts with -the same names, but Spark 4.2 changes internal binary interfaces such as `CatalogManager`. +the same names, but Spark 4.2 and 4.3 change internal binary interfaces such as `CatalogManager` +and `ParserInterface`. For Spark 3.5, you can find the bundled jar in `./paimon-spark/paimon-spark-3.5/target/paimon-spark-3.5_2.12-@@VERSION@@.jar`. -The Spark 4.2 connector can also query [Ossie Semantic Views](../concepts/rest/semantic-views.md#spark-42-ossie-queries) +The Spark 4.2 and 4.3 connectors can also query [Ossie Semantic Views](../concepts/rest/semantic-views.md#spark-42-ossie-queries) registered in a Paimon REST Catalog. ## Add the Connector to Spark diff --git a/paimon-spark/paimon-spark-4.3/pom.xml b/paimon-spark/paimon-spark-4.3/pom.xml new file mode 100644 index 000000000000..b0bcb9afa540 --- /dev/null +++ b/paimon-spark/paimon-spark-4.3/pom.xml @@ -0,0 +1,171 @@ + + + + 4.0.0 + + + org.apache.paimon + paimon-spark + 2.2-SNAPSHOT + + + paimon-spark-4.3_2.13 + Paimon : Spark : 4.3 : 2.13 + + + 4.3.0 + 2.13 + 2.13.18 + + + + + com.networknt + json-schema-validator + 2.0.0 + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + + + org.yaml + snakeyaml + 2.5 + provided + + + org.apache.paimon + paimon-format + + + org.apache.paimon + paimon-spark4-common_${scala.binary.version} + ${project.version} + + + org.apache.spark + spark-sql-api_${scala.binary.version} + ${spark.version} + + + org.apache.spark + spark-connect-shims_${scala.binary.version} + + + org.apache.logging.log4j + log4j-slf4j2-impl + + + + + org.apache.spark + spark-core_${scala.binary.version} + ${spark.version} + + + org.apache.spark + spark-sql_${scala.binary.version} + ${spark.version} + + + org.apache.spark + spark-catalyst_${scala.binary.version} + ${spark.version} + + + org.apache.spark + spark-hive_${scala.binary.version} + ${spark.version} + + + org.apache.spark + spark-sql_${scala.binary.version} + ${spark.version} + tests + test + + + org.apache.spark + spark-connect-shims_${scala.binary.version} + + + + + org.apache.spark + spark-catalyst_${scala.binary.version} + ${spark.version} + tests + test + + + org.apache.spark + spark-core_${scala.binary.version} + ${spark.version} + tests + test + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-paimon + package + + shade + + + + + org.apache.paimon:paimon-spark4-common_${scala.binary.version} + com.networknt:json-schema-validator + com.ethlo.time:itu + + + + + com.networknt + org.apache.paimon.spark.shade.networknt + + + com.ethlo.time + org.apache.paimon.spark.shade.time + + + + + + + + + diff --git a/paimon-spark/paimon-spark-4.3/src/main/resources/META-INF/NOTICE b/paimon-spark/paimon-spark-4.3/src/main/resources/META-INF/NOTICE new file mode 100644 index 000000000000..e616c59a3f17 --- /dev/null +++ b/paimon-spark/paimon-spark-4.3/src/main/resources/META-INF/NOTICE @@ -0,0 +1,12 @@ +paimon-spark-4.3 +Copyright 2023-2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) +- com.networknt:json-schema-validator:2.0.0 +- com.ethlo.time:itu:1.14.0 + +The bundled Ossie JSON Schema is from Apache Ossie (incubating), commit +2692b1cc4708fdcf6a91e597ca928a9bc9d0da60, under the Apache Software License 2.0. diff --git a/paimon-spark/paimon-spark-4.3/src/main/resources/org/apache/paimon/spark/ossie/ossie-schema.json b/paimon-spark/paimon-spark-4.3/src/main/resources/org/apache/paimon/spark/ossie/ossie-schema.json new file mode 100644 index 000000000000..ac15741121b8 --- /dev/null +++ b/paimon-spark/paimon-spark-4.3/src/main/resources/org/apache/paimon/spark/ossie/ossie-schema.json @@ -0,0 +1,396 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/apache/ossie/core-spec/ossie-schema.json", + "title": "Apache Ossie Core Metadata Specification", + "description": "JSON Schema for validating a single Apache Ossie semantic model document", + "type": "object", + "properties": { + "version": { + "type": "string", + "const": "0.3.0.dev0", + "description": "Apache Ossie specification version" + }, + "name": { + "$ref": "#/$defs/SemanticModel/properties/name" + }, + "description": { + "$ref": "#/$defs/SemanticModel/properties/description" + }, + "ai_context": { + "$ref": "#/$defs/SemanticModel/properties/ai_context" + }, + "datasets": { + "$ref": "#/$defs/SemanticModel/properties/datasets" + }, + "relationships": { + "$ref": "#/$defs/SemanticModel/properties/relationships" + }, + "metrics": { + "$ref": "#/$defs/SemanticModel/properties/metrics" + }, + "nonportable_extensions": { + "$ref": "#/$defs/SemanticModel/properties/nonportable_extensions" + } + }, + "required": ["version", "name", "datasets"], + "additionalProperties": false, + "$defs": { + "Dialect": { + "type": "string", + "minLength": 1, + "examples": ["OSSIE_SQL_2026", "ANSI_SQL", "DB_DIALECT", "BI_EXPRESSION", "SEMANTIC_MODEL_EXPRESSION", "PROGRAMMING_LANGUAGE"], + "description": "Expression language a metric or field expression is declared in. Any non-empty string is accepted. OSSIE_SQL_2026 is the only dialect this specification defines; others are opaque to Ossie. See extensions/dialects.md." + }, + "Vendor": { + "type": "string", + "minLength": 1, + "examples": ["BI_TOOL", "DB_VENDOR", "SEMANTIC_LAYER"], + "description": "Vendor name for non-portable extensions. Any non-empty string is accepted. See extensions/vendors.md." + }, + "AIContext": { + "description": "Additional context for AI tools", + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "instructions": { + "type": "string", + "description": "Instructions for AI on how to use this entity" + }, + "synonyms": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Alternative names and terms" + }, + "examples": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Sample questions or use cases" + } + }, + "additionalProperties": true + } + ] + }, + "NonportableExtension": { + "type": "object", + "description": "Vendor-specific state that Ossie carries and does not interpret. Portable behavior does not depend on it.", + "properties": { + "vendor_name": { + "$ref": "#/$defs/Vendor" + }, + "data": { + "type": "object", + "description": "Vendor-specific data as a structured object. The core schema does not reference the structure under data: no reserved key and no common sub-key. Keys are entirely vendor-defined. Values must be JSON-representable. Ossie carries the object but does not interpret it. See core-spec/spec.md#non-portable-extensions." + } + }, + "required": ["vendor_name", "data"], + "additionalProperties": false + }, + "DialectExpression": { + "type": "object", + "description": "Expression in a specific dialect", + "properties": { + "dialect": { + "$ref": "#/$defs/Dialect" + }, + "expression": { + "type": "string", + "minLength": 1, + "description": "SQL or dialect-specific expression" + } + }, + "required": ["dialect", "expression"], + "additionalProperties": false + }, + "Expression": { + "type": "object", + "description": "Expression definition with multi-dialect support", + "properties": { + "sql": { + "type": "string", + "minLength": 1, + "description": "Primary, portable expression written in OSSIE_SQL_2026. When both this field and an OSSIE_SQL_2026 entry in `dialects` are present, this field is authoritative." + }, + "dialects": { + "type": "array", + "items": { + "$ref": "#/$defs/DialectExpression" + }, + "minItems": 1 + } + }, + "anyOf": [ + { "required": ["sql"] }, + { "required": ["dialects"] } + ], + "additionalProperties": false + }, + "DataType": { + "type": "string", + "enum": [ + "String", + "Integer", + "Decimal", + "Float", + "Boolean", + "Date", + "Time", + "DateTime", + "DateTimeTz", + "Opaque" + ], + "description": "Logical data type for fields and metrics, independent of role (e.g. dimension vs fact) and physical representation. `Decimal` is exact base-10 with unspecified precision and scale; `Float` is approximate. `DateTime` has no timezone or offset, while `DateTimeTz` identifies an instant using offset or timezone context but does not guarantee preservation of a named timezone. Omit `datatype` when unknown. `Opaque` means the type is outside the portable vocabulary. A vendor may record its own type name under `nonportable_extensions`; that record is not part of the portable type." + }, + "Dimension": { + "type": "object", + "description": "Dimension metadata", + "properties": { + "is_time": { + "type": "boolean", + "description": "Temporal-role marker. When true, consumers that distinguish time dimensions (e.g. for time-series analysis or temporal filtering) should treat this field as a time dimension. This is a *role* flag, independent of the field's data type: a field with `is_time: true` may carry any `datatype` (e.g. `Integer` for a year grain, `String` for a month name, as well as temporal data types). When `is_time` is unset, it defaults to `true` if `datatype` is one of `Date`, `Time`, `DateTime`, or `DateTimeTz`, and `false` otherwise. Set `is_time: false` explicitly to opt a temporal-typed column (such as an audit timestamp) out of time-dimension treatment." + } + }, + "additionalProperties": false + }, + "Field": { + "type": "object", + "description": "Row-level attribute for grouping, filtering, and metric expressions", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Unique identifier for the field within the dataset" + }, + "expression": { + "$ref": "#/$defs/Expression" + }, + "dimension": { + "$ref": "#/$defs/Dimension" + }, + "label": { + "type": "string", + "description": "Label for categorization" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "datatype": { + "$ref": "#/$defs/DataType" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "nonportable_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/NonportableExtension" + } + } + }, + "required": ["name", "expression"], + "additionalProperties": false + }, + "Dataset": { + "type": "object", + "description": "Logical dataset representing a business entity (fact or dimension table)", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Unique identifier for the dataset" + }, + "source": { + "type": "string", + "minLength": 1, + "description": "Reference to underlying physical table/view (database.schema.table) or query" + }, + "primary_key": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "Dataset fields that uniquely identify rows" + }, + "unique_keys": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "minItems": 1, + "description": "Array of unique key definitions, each a list of dataset fields" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "fields": { + "type": "array", + "items": { + "$ref": "#/$defs/Field" + }, + "description": "The dataset's declared fields. Within the dataset, field names and metric names share one namespace and must all be unique across the two." + }, + "metrics": { + "type": "array", + "items": { + "$ref": "#/$defs/Metric" + }, + "description": "Dataset-scoped metrics: aggregations evaluated at this dataset's row level, for measures that semantically belong to the dataset. The expression references this dataset's declared fields by single-part (unqualified) name, and columns of the dataset's source as source.column. It must resolve entirely within this dataset; an expression that must reference another dataset belongs in the model-scoped metrics instead. Within the dataset, field names and metric names share one namespace and must all be unique across the two." + }, + "nonportable_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/NonportableExtension" + } + } + }, + "required": ["name", "source"], + "additionalProperties": false + }, + "Relationship": { + "type": "object", + "description": "Foreign key relationship between datasets", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Unique identifier for the relationship" + }, + "from": { + "type": "string", + "minLength": 1, + "description": "Dataset on the many side of the relationship" + }, + "to": { + "type": "string", + "minLength": 1, + "description": "Dataset on the one side of the relationship" + }, + "from_columns": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "Array of field names in the 'from' dataset (foreign key fields)" + }, + "to_columns": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "Array of field names in the 'to' dataset (primary or unique key fields)" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "nonportable_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/NonportableExtension" + } + } + }, + "required": ["name", "from", "to", "from_columns", "to_columns"], + "additionalProperties": false + }, + "Metric": { + "type": "object", + "description": "Quantitative measure defined on business data", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Unique identifier for the metric" + }, + "expression": { + "$ref": "#/$defs/Expression" + }, + "description": { + "type": "string", + "description": "Human-readable description of what the metric measures" + }, + "datatype": { + "$ref": "#/$defs/DataType" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "nonportable_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/NonportableExtension" + } + } + }, + "required": ["name", "expression"], + "additionalProperties": false + }, + "SemanticModel": { + "type": "object", + "description": "Top-level container representing a complete semantic model", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Unique identifier for the semantic model" + }, + "description": { + "type": "string", + "description": "Human-readable description" + }, + "ai_context": { + "$ref": "#/$defs/AIContext" + }, + "datasets": { + "type": "array", + "items": { + "$ref": "#/$defs/Dataset" + }, + "minItems": 1, + "description": "Collection of logical datasets" + }, + "relationships": { + "type": "array", + "items": { + "$ref": "#/$defs/Relationship" + }, + "description": "Defines how datasets are connected" + }, + "metrics": { + "type": "array", + "items": { + "$ref": "#/$defs/Metric" + }, + "description": "Model-scoped metrics: aggregations that span datasets or need to be aware of cross-dataset concepts. Expressions reference fields by multi-part name (dataset.field) and may combine fields from more than one dataset. Names must be unique across the model. See core-spec/spec.md#metric-scoping." + }, + "nonportable_extensions": { + "type": "array", + "items": { + "$ref": "#/$defs/NonportableExtension" + } + } + }, + "required": ["name", "datasets"], + "additionalProperties": false + } + } +} diff --git a/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/paimon/spark/ossie/OssieModel.scala b/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/paimon/spark/ossie/OssieModel.scala new file mode 100644 index 000000000000..f133099ddfb3 --- /dev/null +++ b/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/paimon/spark/ossie/OssieModel.scala @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.ossie + +import org.apache.paimon.view.SemanticViewDefinition + +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} +import com.networknt.schema.{SchemaRegistry, SpecificationVersion} +import org.yaml.snakeyaml.{LoaderOptions, Yaml} +import org.yaml.snakeyaml.constructor.SafeConstructor + +import java.nio.charset.StandardCharsets.UTF_8 +import java.util.{Collections, IdentityHashMap, Locale} + +import scala.collection.JavaConverters._ + +case class OssieField(name: String, sql: String) +case class OssieMetric(name: String, sql: String, modelScoped: Boolean) +case class OssieModel( + dataset: String, + source: String, + fields: Seq[OssieField], + metrics: Seq[OssieMetric]) + +/** Read-only execution profile of Ossie 0.3.0.dev0; the original document is never rewritten. */ +object OssieModel { + // The development version is mutable. Keep this schema and the execution profile together. + val schemaCommit: String = "2692b1cc4708fdcf6a91e597ca928a9bc9d0da60" + + private val mapper = new ObjectMapper() + + private lazy val schema = { + val stream = getClass.getResourceAsStream("ossie-schema.json") + require(stream != null, "Missing bundled Ossie schema") + try { + SchemaRegistry.withDefaultDialect(SpecificationVersion.DRAFT_2020_12).getSchema(stream) + } finally { + stream.close() + } + } + + def parse(definition: SemanticViewDefinition): OssieModel = { + require(definition.getFormat == "ossie-yaml", "Semantic query requires format ossie-yaml") + val options = new LoaderOptions() + options.setAllowDuplicateKeys(false) + val document = new Yaml(new SafeConstructor(options)).load[AnyRef](definition.getContent) + require(document != null, "Empty Ossie document") + checkJson(document) + val root = mapper.valueToTree[JsonNode](document) + val errors = schema.validate(root).asScala + require(errors.isEmpty, "Invalid Ossie 0.3.0.dev0 model: " + errors.take(5).mkString("; ")) + val datasets = elements(root, "datasets") + require(datasets.size == 1, "Ossie queries currently require exactly one dataset") + require(elements(root, "relationships").isEmpty, "Ossie relationships are not supported yet") + val dataset = datasets.head + val fields = elements(dataset, "fields").map(f => OssieField(text(f, "name"), expression(f))) + require(fields.nonEmpty, "Ossie queries require declared dataset fields") + val metrics = elements(dataset, "metrics").map( + m => OssieMetric(text(m, "name"), expression(m), modelScoped = false)) ++ + elements(root, "metrics").map( + m => OssieMetric(text(m, "name"), expression(m), modelScoped = true)) + require(metrics.nonEmpty, "Ossie queries require at least one metric") + val names = (fields.map(_.name) ++ metrics.map(_.name)).map(normalize) + require(names.distinct.size == names.size, "Ossie field and metric names must be unique") + val fieldNames = fields.map(f => normalize(f.name)).toSet + val keys = elements(dataset, "primary_key") ++ + elements(dataset, "unique_keys").flatMap(_.elements().asScala) + require( + keys.forall(k => fieldNames.contains(normalize(k.asText()))), + "Ossie keys must reference declared dataset fields") + OssieModel(text(dataset, "name"), text(dataset, "source"), fields, metrics) + } + + def normalize(name: String): String = name.toLowerCase(Locale.ROOT) + + private def expression(node: JsonNode): String = { + val expr = node.get("expression") + if (expr.has("sql")) { + text(expr, "sql") + } else { + val portable = elements(expr, "dialects").filter(d => text(d, "dialect") == "OSSIE_SQL_2026") + require(portable.size == 1, "Ossie query requires an OSSIE_SQL_2026 expression") + text(portable.head, "expression") + } + } + + private def text(node: JsonNode, name: String): String = node.get(name).asText() + + private def elements(node: JsonNode, name: String): Seq[JsonNode] = + Option(node.get(name)).map(_.elements().asScala.toVector).getOrElse(Vector.empty) + + // Preserve YAML scalar types until validation: Jackson's YAML reader turns timestamps into + // strings and map keys into strings, hiding values that Ossie forbids in extension data. + private def checkJson(root: AnyRef): Unit = { + val ancestors = Collections.newSetFromMap(new IdentityHashMap[AnyRef, java.lang.Boolean]()) + // Charge every occurrence, including aliases: a small YAML graph can expand exponentially. + // The budget bounds JSON node count plus UTF-8 scalar/key content before valueToTree allocates. + var remaining = SemanticViewDefinition.MAX_CONTENT_BYTES + def charge(size: Int): Unit = { + require(size <= remaining, "Ossie expanded YAML exceeds the supported size") + remaining -= size + } + def text(value: String): Unit = charge(value.getBytes(UTF_8).length) + def visit(value: AnyRef): Unit = { + charge(1) + value match { + case null | _: java.lang.Boolean => + case s: String => text(s) + case n: java.lang.Double => + require(java.lang.Double.isFinite(n), "Ossie values must be finite JSON numbers") + text(n.toString) + case n: java.lang.Float => + require(java.lang.Float.isFinite(n), "Ossie values must be finite JSON numbers") + text(n.toString) + case n: java.lang.Number => text(n.toString) + case map: java.util.Map[_, _] => + require(ancestors.add(map), "Ossie YAML cannot contain recursive aliases") + try { + map.asScala.foreach { + case (key, item) => + require(key.isInstanceOf[String], "Ossie JSON object keys must be strings") + text(key.asInstanceOf[String]) + visit(item.asInstanceOf[AnyRef]) + } + } finally { ancestors.remove(map) } + case list: java.util.List[_] => + require(ancestors.add(list), "Ossie YAML cannot contain recursive aliases") + try { list.asScala.foreach(item => visit(item.asInstanceOf[AnyRef])) } + finally { ancestors.remove(list) } + case _ => + throw new IllegalArgumentException("Ossie YAML values must be representable as JSON") + } + } + visit(root) + } +} diff --git a/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/catalyst/analysis/Spark42RowLevelCommandRewrite.scala b/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/catalyst/analysis/Spark42RowLevelCommandRewrite.scala new file mode 100644 index 000000000000..1e7468e77339 --- /dev/null +++ b/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/catalyst/analysis/Spark42RowLevelCommandRewrite.scala @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.catalyst.analysis + +import org.apache.paimon.spark.SparkTable +import org.apache.paimon.spark.catalyst.analysis.{MergeSchemaEvolutionHelper, PaimonAssignmentUtils, PaimonRelation} + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.plans.logical.{AnalysisHelper, LogicalPlan, MergeIntoTable, UpdateTable} +import org.apache.spark.sql.catalyst.rules.Rule + +/** Re-enter Spark's native rewrite with fresh, aligned commands that are not marked analyzed. */ +object Spark42RowLevelCommandRewrite + extends Rule[LogicalPlan] + with PureAppendOnlyScope + with MergeSchemaEvolutionHelper { + + override def apply(plan: LogicalPlan): LogicalPlan = { + AnalysisHelper.allowInvokingTransformsInAnalyzer { + plan.transformDown { + case u: UpdateTable if u.resolved && u.rewritable && eligible(u.table) => + val assignments = PaimonAssignmentUtils.alignUpdateAssignments( + u.table.output, + u.assignments, + fromStar = false, + mergeSchemaEnabled = false) + RewriteUpdateTable(u.copy(assignments = assignments)) + + case m: MergeIntoTable + if m.resolved && m.rewritable && !m.withSchemaEvolution && eligible(m.targetTable) => + val relation = PaimonRelation.getPaimonRelation(m.targetTable) + val table = relation.table.asInstanceOf[SparkTable] + val evolved = evolveTargetIfNeeded( + m, + relation, + table, + SparkSession.active, + _.notMatchedBySourceActions) + .map(_._1) + .getOrElse(m) + val aligned = alignAllMergeActions(evolved, evolved.targetTable.output) + RewriteMergeIntoTable(aligned.copy()) + } + } + } + + private def eligible(table: LogicalPlan): Boolean = + targetsV2CopyOnWriteTable(table) || targetsV2DeltaTable(table) +} diff --git a/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/OssieToSparkMetricView.scala b/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/OssieToSparkMetricView.scala new file mode 100644 index 000000000000..6e5e861a3bb5 --- /dev/null +++ b/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/OssieToSparkMetricView.scala @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.paimon + +import org.apache.paimon.spark.ossie.OssieModel +import org.apache.paimon.utils.StringUtils + +import com.fasterxml.jackson.databind.ObjectMapper +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedFunction, UnresolvedStar} +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.parser.CatalystSqlParser +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types._ +import org.apache.spark.unsafe.types.UTF8String + +import scala.collection.mutable + +/** Compile field references before Spark sees the model, preserving source/field shadowing. */ +private[paimon] object OssieToSparkMetricView { + private val aggregates = Set("sum", "count", "avg", "min", "max") + private val functions = aggregates ++ Set("abs", "round", "floor", "ceil", "ceiling", "coalesce") + + def convert( + spark: SparkSession, + model: OssieModel, + source: Seq[String], + sourceColumns: Seq[String]): String = { + val sourceNames = sourceColumns.map(n => OssieModel.normalize(n) -> n).toMap + require(sourceNames.size == sourceColumns.size, "Ambiguous case-insensitive source columns") + val publicNames = + (model.fields.map(_.name) ++ model.metrics.map(_.name)).map(OssieModel.normalize) + val prefix = Iterator + .from(0) + .map(i => s"__ossie_${i}_") + .find(p => !publicNames.exists(_.startsWith(p))) + .get + val fieldNames = model.fields.zipWithIndex.map { + case (f, i) => OssieModel.normalize(f.name) -> s"${prefix}f$i" + }.toMap + val rawColumns = mutable.LinkedHashMap[String, String]() + + def sourceColumn(name: String): String = sourceNames.getOrElse( + OssieModel.normalize(name), + throw new IllegalArgumentException(s"Unknown Ossie source column: $name")) + + def field(name: String): Expression = UnresolvedAttribute( + Seq( + fieldNames.getOrElse( + OssieModel.normalize(name), + throw new IllegalArgumentException(s"Unknown Ossie dataset field: $name")))) + + val projections = model.fields.zipWithIndex.map { + case (f, i) => + val expr = parse(spark, f.sql, metric = false).transformUp { + case a: UnresolvedAttribute => + require(a.nameParts.size == 1, "Ossie field expressions must reference source columns") + UnresolvedAttribute(Seq(sourceColumn(a.nameParts.head))) + } + s"${render(expr)} AS ${quote(s"${prefix}f$i")}" + } + + val measures = model.metrics.map { + m => + val expr = parse(spark, m.sql, metric = true).transformUp { + case a: UnresolvedAttribute => + a.nameParts match { + case Seq(name) if !m.modelScoped => field(name) + case Seq(dataset, name) + if m.modelScoped && + OssieModel.normalize(dataset) == OssieModel.normalize(model.dataset) => + field(name) + case Seq(qualifier, name) + if !m.modelScoped && + qualifier.equalsIgnoreCase("source") && !fieldNames.contains("source") => + val actual = sourceColumn(name) + val alias = rawColumns.getOrElseUpdate(actual, s"${prefix}s${rawColumns.size}") + UnresolvedAttribute(Seq(alias)) + case _ => + throw new IllegalArgumentException(s"Invalid Ossie metric reference: ${a.sql}") + } + } + m.name -> render(expr) + } + val rawProjections = rawColumns.toSeq.map { + case (name, alias) => s"${quote(name)} AS ${quote(alias)}" + } + val sql = "SELECT " + (projections ++ rawProjections).mkString(", ") + + " FROM " + source.map(quote).mkString(".") + val mapper = new ObjectMapper() + val root = mapper.createObjectNode() + root.put("version", "0.1") + root.put("source", sql) + val dimensions = root.putArray("dimensions") + model.fields.zipWithIndex.foreach { + case (f, i) => dimensions.addObject().put("name", f.name).put("expr", quote(s"${prefix}f$i")) + } + val metricNodes = root.putArray("measures") + measures.foreach { + case (name, expr) => metricNodes.addObject().put("name", name).put("expr", expr) + } + // JSON is a YAML subset, and avoids another serialization path for SQL strings. + mapper.writeValueAsString(root) + } + + private def parse(spark: SparkSession, sql: String, metric: Boolean): Expression = { + val conf = new SQLConf() + spark.sessionState.conf.getAllConfs.foreach { case (k, v) => conf.setConfString(k, v) } + conf.setConf(SQLConf.ANSI_ENABLED, true) + conf.setConf(SQLConf.DOUBLE_QUOTED_IDENTIFIERS, true) + conf.setConf(SQLConf.ESCAPED_STRING_LITERALS, false) + val expr = + SQLConf.withExistingConf(conf)(CatalystSqlParser.parseExpression(normalizeStrings(sql))) + var aggregate = false + expr.foreach { + case f: UnresolvedFunction => + require( + f.ignoreNulls.isEmpty && f.orderingWithinGroup.isEmpty, + "Ossie null treatment and ordered-set aggregates are not supported") + require( + f.nameParts.size == 1 && functions.contains(OssieModel.normalize(f.nameParts.head)), + s"Unsupported OSSIE_SQL_2026 function: ${f.nameParts.mkString(".")}" + ) + val name = OssieModel.normalize(f.nameParts.head) + if (aggregates.contains(name)) { + require(metric, "Ossie field expressions cannot contain aggregates") + aggregate = true + } + require( + !f.arguments.exists(_.isInstanceOf[UnresolvedStar]) || name == "count", + "Only COUNT(*) supports a star in Ossie expressions") + case _: Literal | _: UnresolvedAttribute | _: UnresolvedStar => + case _: Add | _: Subtract | _: Multiply | _: Divide | _: Remainder | _: UnaryMinus | + _: UnaryPositive | _: CaseWhen | _: EqualTo | _: LessThan | _: LessThanOrEqual | + _: GreaterThan | _: GreaterThanOrEqual | _: And | _: Or | _: Not | _: IsNull | + _: IsNotNull | _: In => + case c: Cast => + require( + c.dataType.isInstanceOf[NumericType] || c.dataType == StringType || + c.dataType == BooleanType || c.dataType == DateType, + s"Unsupported Ossie cast type: ${c.dataType}" + ) + case other => + throw new IllegalArgumentException(s"Unsupported OSSIE_SQL_2026 expression: ${other.sql}") + } + require(!metric || aggregate, "Ossie metrics must contain a supported aggregate") + expr + } + + // Ossie strings use ANSI doubled quotes and literal backslashes. Normalize only string tokens + // into Spark's escape syntax, leaving typed literals, identifiers and comment boundaries intact. + private def normalizeStrings(sql: String): String = { + val result = new StringBuilder + var pos = 0 + def quoted(delimiter: Char): String = { + val value = new StringBuilder + var closed = false + pos += 1 + while (pos < sql.length && !closed) { + val c = sql.charAt(pos) + pos += 1 + if (c == delimiter) { + if (pos < sql.length && sql.charAt(pos) == delimiter) { + value.append(c) + pos += 1 + } else { closed = true } + } else { value.append(c) } + } + require(closed, "Unterminated quoted Ossie expression") + value.toString + } + while (pos < sql.length) { + val c = sql.charAt(pos) + if (sql.startsWith("--", pos)) { + while (pos < sql.length && sql.charAt(pos) != '\n' && sql.charAt(pos) != '\r') { + pos += 1 + } + result.append(' ') + } else if (sql.startsWith("/*", pos)) { + var depth = 1 + pos += 2 + while (pos < sql.length && depth > 0) { + if (sql.startsWith("/*", pos)) { depth += 1; pos += 2 } + else if (sql.startsWith("*/", pos)) { depth -= 1; pos += 2 } + else { pos += 1 } + } + require(depth == 0, "Unterminated Ossie SQL comment") + result.append(' ') + } else if (c == '"') { + result.append(quote(quoted(c))) + } else if (c == '`') { + val start = pos + quoted(c) + result.append(sql.substring(start, pos)) + } else if (c == '\'') { + val rawPrefix = pos > 0 && (sql.charAt(pos - 1) == 'r' || sql.charAt(pos - 1) == 'R') && + (pos == 1 || !Character.isJavaIdentifierPart(sql.charAt(pos - 2))) + require(!rawPrefix, "Ossie requires ANSI string literals") + val value = quoted(c) + result.append('\'') + value.foreach { + case '\\' => result.append("\\\\") + case '\'' => result.append("\\'") + case other => result.append(other) + } + result.append('\'') + } else { + result.append(c) + pos += 1 + } + } + result.toString + } + + private def quote(name: String): String = "`" + name.replace("`", "``") + "`" + + // UnresolvedFunction inherits Expression.sql, which drops DISTINCT and FILTER. Render those + // explicitly before asking the surrounding scalar expressions to render their own syntax. + private def render(expr: Expression): String = expr.transformUp { + case l: Literal if l.value != null && l.dataType.isInstanceOf[StringType] => + val hex = StringUtils.byteToHexString(l.value.asInstanceOf[UTF8String].getBytes) + // Native Metric View parsing uses session settings; binary literals avoid a second round + // of string escaping regardless of escapedStringLiterals. + RenderedSQL(s"CAST(X'$hex' AS STRING)") + case f: UnresolvedFunction => + val distinct = if (f.isDistinct) "DISTINCT " else "" + val filter = f.filter.map(p => s" FILTER (WHERE ${p.sql})").getOrElse("") + RenderedSQL( + s"${f.nameParts.map(quote).mkString(".")}($distinct${f.arguments.map(_.sql).mkString(", ")})$filter") + }.sql + + private case class RenderedSQL(text: String) extends LeafExpression with Unevaluable { + override def dataType: org.apache.spark.sql.types.DataType = NullType + override def nullable: Boolean = true + override def sql: String = text + } +} diff --git a/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/PaimonSemanticViewResolver.scala b/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/PaimonSemanticViewResolver.scala new file mode 100644 index 000000000000..d3a0ca25ecc9 --- /dev/null +++ b/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/PaimonSemanticViewResolver.scala @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.paimon + +import org.apache.paimon.catalog.{DelegateCatalog, Identifier => PaimonIdentifier} +import org.apache.paimon.rest.RESTCatalog +import org.apache.paimon.rest.exceptions.NoSuchResourceException +import org.apache.paimon.spark.SparkCatalog +import org.apache.paimon.spark.ossie.OssieModel +import org.apache.paimon.table.FileStoreTable +import org.apache.paimon.view.SemanticViewDefinition + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation +import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType} +import org.apache.spark.sql.catalyst.plans.logical.{CTERelationDef, CTERelationRef, Join, LateralJoin, LogicalPlan, SubqueryAlias} +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.connector.catalog.PaimonLookupCatalog +import org.apache.spark.sql.execution.command.MetricViewHelper +import org.apache.spark.sql.metricview.util.MetricViewPlanner + +import scala.collection.JavaConverters._ +import scala.collection.mutable + +/** Resolve REST semantic definitions through Spark's Metric View engine without persisting a copy. */ +class PaimonSemanticViewResolver(spark: SparkSession) + extends Rule[LogicalPlan] + with PaimonLookupCatalog { + + protected lazy val catalogManager = spark.sessionState.catalogManager + + override def apply(plan: LogicalPlan): LogicalPlan = { + val definitions = mutable.Map[(RESTCatalog, PaimonIdentifier), Option[SemanticViewDefinition]]() + plan.resolveOperatorsDown { + case u @ UnresolvedRelation( + parts @ CatalogAndIdentifier(catalog: SparkCatalog, ident), + _, + _) => + DelegateCatalog.rootCatalog(catalog.paimonCatalog()) match { + // An ordinary view expanded by the preceding rule may introduce unresolved source + // tables after Spark's relation resolution has run. Leave those for the next iteration. + case rest: RESTCatalog if !catalog.tableExists(ident) => + val id = PaimonIdentifier.create(ident.namespace().head, ident.name()) + val definition = definitions.getOrElseUpdate( + (rest, id), + try { + Some( + rest + .semanticViewManagement() + .getSemanticView(id) + .getDefinition) + } catch { + case _: NoSuchResourceException => None + }) + definition + .map { + definition => + require(!u.isStreaming, "Ossie Semantic Views support batch queries only") + require( + u.options.get(UnresolvedRelation.REQUIRED_WRITE_PRIVILEGES) == null, + "Ossie Semantic Views are read-only") + require( + !isJoined(plan, u), + "Joins against Ossie Semantic Views are not supported yet") + val model = OssieModel.parse(definition) + val source = + spark.sessionState.sqlParser.parseMultipartIdentifier(model.source) match { + case Seq(table) => Seq(catalog.name(), ident.namespace().head, table) + case Seq(db, table) => Seq(catalog.name(), db, table) + case Seq(cat, db, table) if cat == catalog.name() => Seq(cat, db, table) + case _ => + throw new IllegalArgumentException( + "Ossie source must be a table in the same Paimon catalog") + } + val table = + catalog.paimonCatalog().getTable(PaimonIdentifier.create(source(1), source(2))) + require( + table.isInstanceOf[FileStoreTable], + "Ossie source must be a physical Paimon table") + val yaml = OssieToSparkMetricView.convert( + spark, + model, + source, + table.rowType().getFieldNames.asScala.toSeq) + val (analyzed, _) = MetricViewHelper.analyzeMetricViewText(spark, parts, yaml) + val metadata = CatalogTable( + identifier = TableIdentifier( + ident.name(), + Some(ident.namespace().head), + Some(catalog.name())), + tableType = CatalogTableType.VIEW, + storage = CatalogStorageFormat.empty, + schema = analyzed.schema, + viewText = Some(yaml) + ) + SubqueryAlias( + parts, + MetricViewPlanner + .planRead(metadata, yaml, spark.sessionState.sqlParser, analyzed.schema)) + } + .getOrElse(u) + case _ => u + } + } + } + + private def isJoined(plan: LogicalPlan, relation: UnresolvedRelation): Boolean = { + val ctes = plan.collectWithSubqueries { case cte: CTERelationDef => cte } + val semanticCtes = mutable.Set.empty[Long] + def dependsOnSemantic(node: LogicalPlan): Boolean = + node.collectFirstWithSubqueries { + case n if n eq relation => true + case ref: CTERelationRef if semanticCtes.contains(ref.cteId) => true + }.isDefined + + // CTE definitions are siblings of their consumers, and can reference other CTEs. Resolve + // those dependencies before inspecting joins, including joins inside scalar subqueries. + var changed = true + while (changed) { + val previousSize = semanticCtes.size + ctes.foreach(cte => if (dependsOnSemantic(cte.child)) { semanticCtes += cte.id }) + changed = semanticCtes.size != previousSize + } + plan.collectFirstWithSubqueries { + case join: Join if dependsOnSemantic(join) => true + case join: LateralJoin if dependsOnSemantic(join) => true + }.isDefined + } +} diff --git a/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/data/SparkInternalRowVariantTest.scala b/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/data/SparkInternalRowVariantTest.scala new file mode 100644 index 000000000000..fad39f3da941 --- /dev/null +++ b/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/data/SparkInternalRowVariantTest.scala @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.data + +import org.apache.paimon.data.GenericRow +import org.apache.paimon.data.variant.GenericVariant +import org.apache.paimon.types.{RowType, VariantType} + +import org.apache.spark.SparkFunSuite +import org.apache.spark.sql.types.DataTypes +import org.apache.spark.unsafe.types.VariantVal + +class SparkInternalRowVariantTest extends SparkFunSuite { + + test("get variant with generic data type access") { + val variant = GenericVariant.fromJson("""{"id":1}""") + val rowType = RowType.of(new VariantType()) + val row = SparkInternalRow.create(rowType).replace(GenericRow.of(variant)) + + val actual = row.get(0, DataTypes.VariantType).asInstanceOf[VariantVal] + + assert(actual.getValue.sameElements(variant.value())) + assert(actual.getMetadata.sameElements(variant.metadata())) + } + + test("get variant with typed access") { + val variant = GenericVariant.fromJson("""{"id":1}""") + val rowType = RowType.of(new VariantType()) + val row = SparkInternalRow.create(rowType).replace(GenericRow.of(variant)) + + // `getVariant` is implemented by `Spark4InternalRow`, while the generic `get` above goes + // through `Spark4Shim.toSparkVariant`. + val actual = row.getVariant(0) + + assert(actual.getValue.sameElements(variant.value())) + assert(actual.getMetadata.sameElements(variant.metadata())) + } +} diff --git a/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/ossie/OssieModelTest.scala b/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/ossie/OssieModelTest.scala new file mode 100644 index 000000000000..da267139fe77 --- /dev/null +++ b/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/ossie/OssieModelTest.scala @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.ossie + +import org.apache.paimon.view.SemanticViewDefinition + +import org.scalatest.funsuite.AnyFunSuite + +class OssieModelTest extends AnyFunSuite { + private val model = + """version: 0.3.0.dev0 + |name: revenue + |datasets: + | - name: orders + | source: orders + | fields: + | - name: amount + | expression: {sql: price} + | metrics: + | - name: revenue + | expression: {sql: SUM(amount)} + |""".stripMargin + + private def parse(text: String): OssieModel = + OssieModel.parse(new SemanticViewDefinition("ossie-yaml", text)) + + test("parse the flat document and ignore preserved nonportable metadata") { + val text = model + + "nonportable_extensions:\n - vendor_name: example\n data: {nested: [1, true, null]}\n" + val result = parse(text) + assert(result.dataset == "orders") + assert(result.fields == Seq(OssieField("amount", "price"))) + assert(result.metrics == Seq(OssieMetric("revenue", "SUM(amount)", modelScoped = false))) + val portable = parse( + model.replace( + "{sql: SUM(amount)}", + "{dialects: [{dialect: OSSIE_SQL_2026, expression: SUM(amount)}]}")) + assert(portable == result) + } + + test("validate the pinned schema and reject old document shapes and unknown core fields") { + Seq( + model.replace("version: 0.3.0.dev0", "version: 0.2.0.dev0"), + model + "semantic_model: []\n", + model + "custom_extensions: []\n", + model.replace("expression: {sql: price}", "expression: {sql: 42}"), + model.replace("source: orders", "source: orders\n primary_key: [missing]"), + model.replace("name: revenue\n expression", "name: AMOUNT\n expression") + ).foreach(text => intercept[IllegalArgumentException](parse(text))) + } + + test("reject multiple datasets and relationships instead of flattening joins") { + // Add the second dataset before the first dataset's fields, retaining a valid document. + val second = + model.replace("datasets:\n", "datasets:\n - name: customers\n source: customers\n") + assert(intercept[IllegalArgumentException](parse(second)).getMessage.contains("exactly one")) + val relationship = model + "relationships:\n - name: self\n from: orders\n" + + " to: orders\n from_columns: [amount]\n to_columns: [amount]\n" + assert( + intercept[IllegalArgumentException](parse(relationship)).getMessage.contains("relationships")) + } + + test("reject duplicate YAML keys, trailing documents, and nonfinite extension values") { + Seq( + model + "name: duplicate\n", + model + "---\n" + model, + model + "nonportable_extensions:\n - vendor_name: test\n data: {value: .inf}\n" + ).foreach(text => intercept[Exception](parse(text))) + } + + test("reject dialect-only expressions without a portable form") { + val text = model.replace( + "{sql: SUM(amount)}", + "{dialects: [{dialect: UNKNOWN, expression: SUM(amount)}]}") + assert(intercept[IllegalArgumentException](parse(text)).getMessage.contains("OSSIE_SQL_2026")) + assert( + intercept[IllegalArgumentException]( + OssieModel.parse(new SemanticViewDefinition("databricks-yaml", model))).getMessage + .contains("ossie-yaml")) + } + + test("extension data must be JSON values with string keys") { + Seq( + "2026-11-13", + "2026-11-13T01:02:03Z", + ".nan", + "!!binary SGVsbG8=", + "!!set {a: null}", + "&cycle [*cycle]") + .foreach { + value => + val text = + model + s"nonportable_extensions:\n - vendor_name: test\n data: {value: $value}\n" + intercept[Exception](parse(text)) + } + intercept[Exception]( + parse(model + + "nonportable_extensions:\n - vendor_name: test\n data: {123: value}\n")) + assert(parse(model + + "nonportable_extensions:\n - vendor_name: test\n data: {date: '2026-11-13', nested: [null, true, 1]}\n").dataset == "orders") + } + + test("bound expanded alias graphs and repeated scalar aliases") { + val graph = " p0: &p0 [0]\n" + (1 to 20).map { + i => s" p$i: &p$i [*p${i - 1}, *p${i - 1}]\n" + }.mkString + val scalar = " text: &text '" + ("x" * 4096) + "'\n copies: [" + + Seq.fill(300)("*text").mkString(", ") + "]\n" + Seq(graph, scalar).foreach { + data => + val text = model + "nonportable_extensions:\n - vendor_name: test\n data:\n" + data + assert(text.length < SemanticViewDefinition.MAX_CONTENT_BYTES) + val error = intercept[IllegalArgumentException](parse(text)) + assert(error.getMessage.contains("expanded YAML")) + } + val ordinaryAlias = model + + "nonportable_extensions:\n - vendor_name: test\n data: {original: &value [1, null], copy: *value}\n" + assert(parse(ordinaryAlias).dataset == "orders") + } +} diff --git a/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/sql/Spark43CompatibilityTest.scala b/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/sql/Spark43CompatibilityTest.scala new file mode 100644 index 000000000000..0fb254110917 --- /dev/null +++ b/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/sql/Spark43CompatibilityTest.scala @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.sql + +import org.apache.paimon.rest.RESTCatalogServer +import org.apache.paimon.rest.auth.BearTokenAuthProvider +import org.apache.paimon.rest.responses.ConfigResponse +import org.apache.paimon.spark.SparkCatalog +import org.apache.paimon.spark.catalyst.parser.extensions.PaimonSpark4SqlExtensionsParser +import org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions + +import org.apache.spark.SparkConf +import org.apache.spark.sql.{QueryTest, Row} +import org.apache.spark.sql.test.SharedSparkSession + +import java.nio.file.Files +import java.util.Collections + +/** Exercise the APIs whose signatures changed in Spark 4.2 and 4.3 using real Paimon files. */ +class Spark43CompatibilityTest extends QueryTest with SharedSparkSession { + private val warehouse = Files.createTempDirectory("paimon-spark43-").toFile + private lazy val server = { + val s = new RESTCatalogServer( + warehouse.toURI.toString, + new BearTokenAuthProvider("test-token"), + new ConfigResponse(Collections.singletonMap("prefix", "paimon"), Collections.emptyMap()), + "spark43-tests" + ) + s.start() + s + } + + override protected def sparkConf: SparkConf = super.sparkConf + .set("spark.sql.catalog.paimon", classOf[SparkCatalog].getName) + .set("spark.sql.catalog.paimon.metastore", "rest") + .set("spark.sql.catalog.paimon.uri", server.getUrl) + .set("spark.sql.catalog.paimon.warehouse", "spark43-tests") + .set("spark.sql.catalog.paimon.token.provider", "bear") + .set("spark.sql.catalog.paimon.token", "test-token") + .set("spark.sql.defaultCatalog", "paimon") + .set("spark.sql.extensions", classOf[PaimonSparkSessionExtensions].getName) + // Spark 4.3 dual-runs the single-pass resolver in tests, and it cannot resolve Paimon views. + .set("spark.sql.analyzer.singlePassResolver.dualRunWithLegacy", "false") + + override protected def beforeAll(): Unit = { + super.beforeAll() + sql("CREATE DATABASE paimon.test") + sql("USE paimon.test") + } + + override protected def afterAll(): Unit = { + try { super.afterAll() } + finally { + server.shutdown() + org.apache.commons.io.FileUtils.deleteDirectory(warehouse) + } + } + + test("create, insert, CTAS and static and dynamic partition overwrite") { + withTable("t", "copied") { + sql( + "CREATE TABLE t (id INT, label VARCHAR(20), pt STRING) PARTITIONED BY (pt) " + + "TBLPROPERTIES ('bucket'='-1', 'file.format'='avro')") + sql("INSERT INTO t VALUES (1, 'a', 'x'), (2, 'b', 'y')") + sql("INSERT OVERWRITE t PARTITION (pt='x') SELECT 3, 'c'") + checkAnswer(sql("SELECT * FROM t"), Seq(Row(3, "c", "x"), Row(2, "b", "y"))) + withSQLConf("spark.sql.sources.partitionOverwriteMode" -> "dynamic") { + sql("INSERT OVERWRITE t SELECT 4, 'd', 'y'") + } + checkAnswer(sql("SELECT * FROM t"), Seq(Row(3, "c", "x"), Row(4, "d", "y"))) + sql("CREATE TABLE copied TBLPROPERTIES ('file.format'='avro') AS SELECT * FROM t") + checkAnswer(sql("SELECT * FROM copied"), sql("SELECT * FROM t")) + } + } + + test("create or replace table as select") { + withTable("t") { + sql("CREATE TABLE t (id INT, label STRING) TBLPROPERTIES ('file.format'='avro')") + sql("INSERT INTO t VALUES (1, 'old')") + sql( + "CREATE OR REPLACE TABLE t TBLPROPERTIES ('file.format'='avro') " + + "AS SELECT 2 AS id, 'new' AS label") + checkAnswer(sql("SELECT * FROM t"), Seq(Row(2, "new"))) + } + } + + test("replace table with a column list") { + withTable("t") { + sql("CREATE TABLE t (id INT, label STRING) TBLPROPERTIES ('file.format'='avro')") + sql("INSERT INTO t VALUES (1, 'old')") + sql("CREATE OR REPLACE TABLE t (id INT, score DOUBLE) TBLPROPERTIES ('file.format'='avro')") + sql("INSERT INTO t VALUES (2, 0.5)") + checkAnswer(sql("SELECT * FROM t"), Seq(Row(2, 0.5))) + } + } + + test("split statements through the Paimon parser") { + val parser = spark.sessionState.sqlParser + assert(parser.isInstanceOf[PaimonSpark4SqlExtensionsParser]) + val result = parser.splitStatements("SELECT ';'; SELECT 2;") + assert(result.completeStatements.map(_.statement.trim) === Seq("SELECT ';'", "SELECT 2")) + assert(result.partialStatement.isEmpty) + } + + test("ordinary views, table description and partition description") { + withTable("t") { + sql( + "CREATE TABLE t (id INT, pt STRING) PARTITIONED BY (pt) " + + "TBLPROPERTIES ('bucket'='-1', 'file.format'='avro')") + sql("INSERT INTO t VALUES (1, 'x')") + try { + sql("CREATE VIEW v AS SELECT id FROM t") + checkAnswer(sql("SELECT * FROM v"), Seq(Row(1))) + assert(sql("DESCRIBE t").collect().exists(_.getString(0) == "id")) + assert( + sql("DESCRIBE EXTENDED t PARTITION (pt='x')") + .collect() + .exists(_.getString(0) == "# Detailed Partition Information")) + } finally { sql("DROP VIEW IF EXISTS v") } + } + } + + test("SQL scalar functions, builtin lookup and unsupported table function rejection") { + try { + sql("CREATE FUNCTION add_one(x INT) RETURNS INT RETURN x + 1") + checkAnswer( + sql("SELECT add_one(4), SUM(abs(v)) FROM VALUES (-2), (3) AS t(v)"), + Seq(Row(5, 5))) + val unsupported = intercept[UnsupportedOperationException] { + sql("CREATE FUNCTION rows_plus(x INT) RETURNS TABLE (v INT) RETURN SELECT x + 1") + } + assert(unsupported.getMessage.contains("SQL table functions")) + } finally { + sql("DROP FUNCTION IF EXISTS add_one") + sql("DROP FUNCTION IF EXISTS rows_plus") + } + } + + test("unaliased v1 function call keeps the database-qualified column name") { + try { + sql("CREATE FUNCTION abs_udf AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFAbs'") + val df = sql("SELECT abs_udf(-3)") + checkAnswer(df, Seq(Row(3))) + assert(df.columns.toSeq === Seq("test.abs_udf(-3)")) + } finally { sql("DROP FUNCTION IF EXISTS abs_udf") } + } + + private val modes: Seq[(String, String)] = Seq( + "primary-key" -> "'bucket'='1', 'primary-key'='id'", + "append" -> "'bucket'='-1'", + "row-tracking" -> "'bucket'='-1', 'row-tracking.enabled'='true'", + "deletion-vectors" -> "'bucket'='-1', 'deletion-vectors.enabled'='true'", + "data-evolution" -> + ("'bucket'='-1', 'row-tracking.enabled'='true', 'data-evolution.enabled'='true', " + + "'deletion-vectors.enabled'='true'") + ) + + for ((mode, properties) <- modes; v2 <- Seq(false, true)) { + test(s"UPDATE, DELETE and MERGE on $mode tables with v2 writes $v2") { + withSQLConf("spark.paimon.write.use-v2-write" -> v2.toString) { + withTable("t") { + sql(s"CREATE TABLE t (id INT, v INT) TBLPROPERTIES ($properties, 'file.format'='avro')") + sql("INSERT INTO t VALUES (1, 10), (2, 20), (3, 30)") + sql("UPDATE t SET v=11 WHERE id=1") + checkAnswer(sql("SELECT * FROM t"), Seq(Row(1, 11), Row(2, 20), Row(3, 30))) + sql("DELETE FROM t WHERE id=2") + sql( + "MERGE INTO t USING (SELECT 3 AS id, 33 AS v UNION ALL SELECT 4, 44) s " + + "ON t.id=s.id WHEN MATCHED THEN UPDATE SET * " + + "WHEN NOT MATCHED THEN INSERT *") + checkAnswer(sql("SELECT * FROM t"), Seq(Row(1, 11), Row(3, 33), Row(4, 44))) + } + } + } + } + + test("native V2 rewrite handles subquery updates and insert-only merges") { + withSQLConf("spark.paimon.write.use-v2-write" -> "true") { + withTable("t") { + sql( + "CREATE TABLE t (id INT, v INT) " + + "TBLPROPERTIES ('bucket'='-1', 'file.format'='avro')") + sql("INSERT INTO t VALUES (1, 10), (2, 20)") + sql("UPDATE t SET v=99 WHERE id IN (SELECT 1)") + sql( + "MERGE INTO t USING (SELECT 3 AS id, 30 AS v) s ON t.id=s.id " + + "WHEN NOT MATCHED THEN INSERT *") + checkAnswer(sql("SELECT * FROM t"), Seq(Row(1, 99), Row(2, 20), Row(3, 30))) + sql("DELETE FROM t") + checkAnswer(sql("SELECT COUNT(*) FROM t"), Seq(Row(0L))) + } + } + } + + test("variant values and spatial scalar and array getters") { + val point = "0101000000000000000000F03F0000000000000040" + withTable("typed") { + sql( + "CREATE TABLE typed (v VARIANT, g GEOMETRY(3857), " + + "a ARRAY, gg GEOGRAPHY(4326)) " + + "TBLPROPERTIES ('bucket'='-1', 'file.format'='parquet')") + sql( + s"INSERT INTO typed SELECT parse_json('{\"id\":1}'), " + + s"ST_GeomFromWKB(unhex('$point'), 3857), " + + s"array(ST_GeomFromWKB(unhex('$point'), 3857)), ST_GeogFromWKB(unhex('$point'))") + checkAnswer( + sql( + "SELECT variant_get(v, '$.id', 'int'), hex(ST_AsBinary(g)), " + + "hex(ST_AsBinary(a[0])), hex(ST_AsBinary(gg)), ST_Srid(g), ST_Srid(a[0]) FROM typed"), + Seq(Row(1, point, point, point, 3857, 3857)) + ) + } + } +} diff --git a/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala b/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala new file mode 100644 index 000000000000..42ea9561e3e1 --- /dev/null +++ b/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala @@ -0,0 +1,497 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.paimon + +import org.apache.paimon.catalog.{DelegateCatalog, Identifier} +import org.apache.paimon.management.SemanticViewManagement +import org.apache.paimon.rest.{RESTApi, RESTCatalog, RESTCatalogServer} +import org.apache.paimon.rest.auth.{AuthProvider, BearTokenAuthProvider} +import org.apache.paimon.rest.requests.UpsertSemanticViewRequest +import org.apache.paimon.rest.responses.{ConfigResponse, GetSemanticViewResponse} +import org.apache.paimon.spark.SparkCatalog +import org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions +import org.apache.paimon.view.SemanticViewDefinition + +import okhttp3.mockwebserver.{Dispatcher, MockResponse, RecordedRequest} +import org.apache.spark.SparkConf +import org.apache.spark.sql.{QueryTest, Row} +import org.apache.spark.sql.test.SharedSparkSession + +import java.nio.file.Files +import java.util.Collections +import java.util.concurrent.ConcurrentHashMap + +/** Real Spark queries, real REST client, and physical Paimon source files. */ +class OssieSemanticViewTest extends QueryTest with SharedSparkSession { + private val id = Identifier.create("sales", "order_metrics") + private val directory = Files.createTempDirectory("paimon-ossie-").toFile + private lazy val server = { + val s = new SemanticServer(directory.toURI.toString) + s.start() + s + } + private def models: SemanticViewManagement = { + val catalog = spark.sessionState.catalogManager.catalog("paimon").asInstanceOf[SparkCatalog] + DelegateCatalog + .rootCatalog(catalog.paimonCatalog()) + .asInstanceOf[RESTCatalog] + .semanticViewManagement() + } + + override protected def sparkConf: SparkConf = super.sparkConf + .set("spark.sql.catalog.paimon", classOf[SparkCatalog].getName) + .set("spark.sql.catalog.paimon.metastore", "rest") + .set("spark.sql.catalog.paimon.uri", server.getUrl) + .set("spark.sql.catalog.paimon.warehouse", "ossie-tests") + .set("spark.sql.catalog.paimon.token.provider", "bear") + .set("spark.sql.catalog.paimon.token", "test-token") + .set("spark.sql.catalog.paimon.cache-enabled", "false") + .set("spark.sql.extensions", classOf[PaimonSparkSessionExtensions].getName) + // Spark 4.3 dual-runs the single-pass resolver in tests, and it cannot resolve Paimon views. + .set("spark.sql.analyzer.singlePassResolver.dualRunWithLegacy", "false") + + override protected def beforeAll(): Unit = { + super.beforeAll() + spark.sql("CREATE DATABASE paimon.sales") + spark.sql( + "CREATE TABLE paimon.sales.orders (region_code STRING, amount DECIMAL(12, 2), " + + "discount DECIMAL(12, 2)) TBLPROPERTIES ('bucket'='-1', 'file.format'='avro')") + spark.sql( + "INSERT INTO paimon.sales.orders VALUES ('CN', 100, 10), ('CN', 100, 20), ('SG', 80, 20)") + } + + override protected def afterAll(): Unit = { + try { + super.afterAll() + } finally { + server.shutdown() + org.apache.spark.util.Utils.deleteRecursively(directory) + } + } + + private def register(yaml: String = model): Unit = { + server.semanticStatus = 200 + server.denySource = false + models.upsertSemanticView(id, new SemanticViewDefinition("ossie-yaml", yaml)) + } + + test("query dataset and model metrics with field/source shadowing and distinct aggregates") { + register() + checkAnswer( + spark.sql( + "SELECT region, MEASURE(revenue), MEASURE(gross), MEASURE(orders), " + + "MEASURE(average), MEASURE(distinct_amounts), MEASURE(model_revenue) " + + "FROM paimon.sales.order_metrics GROUP BY region"), + Seq( + Row("CN", BigDecimal(170), BigDecimal(200), 2L, BigDecimal(85), 1L, BigDecimal(170)), + Row("SG", BigDecimal(60), BigDecimal(80), 1L, BigDecimal(60), 1L, BigDecimal(60)) + ) + ) + checkAnswer( + spark.sql( + "SELECT MEASURE(revenue), MEASURE(orders) " + + "FROM paimon.sales.order_metrics WHERE region = 'CN'"), + Seq(Row(BigDecimal(170), 2L))) + checkAnswer( + spark.sql("SELECT MEASURE(revenue) FROM paimon.sales.order_metrics"), + Seq(Row(BigDecimal(230)))) + assert(server.authorizedSemanticReads > 0) + checkAnswer( + spark.sql( + "SELECT MEASURE(revenue), MEASURE(orders) " + + "FROM paimon.sales.order_metrics WHERE region = 'missing'"), + Seq(Row(null, 0L))) + } + + test("public field names cannot shadow private metric input aliases") { + Seq("__ossie_f1", "__ossie_0_f1").foreach { + name => + register(model.replace("name: region", s"name: $name")) + checkAnswer( + spark.sql( + s"SELECT MEASURE(revenue) FROM paimon.sales.order_metrics " + + s"WHERE $name = 'CN'"), + Seq(Row(BigDecimal(170)))) + } + } + + test("preserve FILTER, quoted identifiers, true division, and half-away-from-zero rounding") { + register( + model + .replace("sql: SUM(amount)", "sql: ROUND(SUM(amount) / COUNT(*), 0)") + .replace( + "sql: COUNT(DISTINCT source.amount)", + "sql: COUNT(DISTINCT source.amount) FILTER (WHERE region = 'CN')") + .replace( + "sql: AVG(amount)", + "sql: AVG(CASE WHEN \"region\" = 'CN' THEN \"amount\" ELSE NULL END)") + .replace("sql: SUM(source.amount)", "sql: ROUND(MIN(CAST(-2.5 AS DECIMAL(3, 1))), 0)") + .replace("sql: SUM(orders.amount)", "sql: COUNT(*) / COUNT(DISTINCT orders.region)")) + checkAnswer( + spark.sql("SELECT MEASURE(revenue), MEASURE(distinct_amounts), " + + "MEASURE(average), MEASURE(gross), MEASURE(model_revenue) FROM paimon.sales.order_metrics"), + Seq(Row(BigDecimal(77), 1L, BigDecimal(85), BigDecimal(-3), 1.5d)) + ) + } + + test("new analysis sees REST replacement and is independent of the current database") { + register() + checkAnswer( + spark.sql("SELECT MEASURE(revenue) FROM paimon.sales.order_metrics"), + Seq(Row(BigDecimal(230)))) + register(model.replace("sql: SUM(amount)", "sql: MAX(amount)")) + withSQLConf("spark.sql.caseSensitive" -> "true") { + checkAnswer( + spark.sql("SELECT MEASURE(revenue) FROM paimon.sales.order_metrics"), + Seq(Row(BigDecimal(90)))) + } + checkAnswer(spark.sql("SELECT SUM(amount) FROM paimon.sales.orders"), Seq(Row(BigDecimal(280)))) + spark.sql( + "CREATE OR REPLACE VIEW paimon.sales.ordinary AS SELECT region_code FROM paimon.sales.orders") + checkAnswer( + spark.sql("SELECT * FROM paimon.sales.ordinary"), + Seq(Row("CN"), Row("CN"), Row("SG"))) + assert(spark.sql("DESCRIBE paimon.sales.orders").collect().exists(_.getString(0) == "amount")) + spark.sql("CREATE OR REPLACE FUNCTION paimon.sales.add_one(x INT) RETURNS INT RETURN x + 1") + checkAnswer(spark.sql("SELECT paimon.sales.add_one(4)"), Seq(Row(5))) + } + + test("portable string literals keep backslashes and apostrophes under either Spark setting") { + Seq(false, true).foreach { + escaped => + withSQLConf("spark.sql.parser.escapedStringLiterals" -> escaped.toString) { + Seq("C:\\new\\tab\\%\\path's", "O'Brien", "", "路径\\new").foreach { + value => + val literal = "'" + value.replace("'", "''") + "'" + register( + model + .replace("sql: region_code", s"sql: |-\n $literal") + .replace( + "sql: COUNT(DISTINCT source.amount)", + s"sql: COUNT(*) FILTER (WHERE region = $literal)")) + checkAnswer( + spark.sql("SELECT region, MEASURE(revenue), MEASURE(distinct_amounts) " + + "FROM paimon.sales.order_metrics GROUP BY region"), + Seq(Row(value, BigDecimal(230), 3L)) + ) + } + register(model + .replace("sql: region_code", "sql: |-\n CAST(DATE '2026-11-13' AS STRING)")) + checkAnswer( + spark.sql( + "SELECT region, MEASURE(revenue) FROM paimon.sales.order_metrics GROUP BY region"), + Seq(Row("2026-11-13", BigDecimal(230)))) + register( + model.replace( + "sql: region_code", + "sql: |-\n CASE WHEN \"region_code\" = 'CN' /* ' ignored */ " + + "THEN 'O''Brien' -- ' ignored\n ELSE '路径\\new' END" + )) + checkAnswer( + spark.sql( + "SELECT region, MEASURE(revenue) FROM paimon.sales.order_metrics GROUP BY region"), + Seq(Row("O'Brien", BigDecimal(170)), Row("路径\\new", BigDecimal(60))) + ) + } + } + } + + test("CTE aliases and lateral syntax cannot hide unsupported semantic joins") { + register() + Seq( + "SELECT m.region, MEASURE(m.revenue) FROM paimon.sales.order_metrics m " + + "JOIN LATERAL (SELECT 1 AS x) t ON true GROUP BY m.region", + "WITH m AS (SELECT * FROM paimon.sales.order_metrics) " + + "SELECT MEASURE(m.revenue) FROM m JOIN paimon.sales.orders o " + + "ON m.region=o.region_code GROUP BY m.region", + "WITH m AS (SELECT * FROM paimon.sales.order_metrics), n AS (SELECT * FROM m) " + + "SELECT MEASURE(n.revenue) FROM n JOIN paimon.sales.orders o " + + "ON n.region=o.region_code GROUP BY n.region", + "WITH m AS (SELECT * FROM paimon.sales.order_metrics) " + + "SELECT (SELECT MEASURE(m.revenue) FROM m JOIN paimon.sales.orders o " + + "ON m.region=o.region_code) AS revenue" + ).foreach { + query => + val error = intercept[Exception](spark.sql(query).collect()) + assert(error.toString.contains("Joins against Ossie"), error.toString) + } + checkAnswer( + spark.sql( + "WITH m AS (SELECT MEASURE(revenue) AS revenue " + + "FROM paimon.sales.order_metrics) SELECT revenue FROM m"), + Seq(Row(BigDecimal(230)))) + checkAnswer( + spark.sql( + "WITH ordinary AS (SELECT o.region_code FROM paimon.sales.orders o " + + "JOIN paimon.sales.orders p ON o.region_code=p.region_code) " + + "SELECT MEASURE(revenue) FROM paimon.sales.order_metrics"), + Seq(Row(BigDecimal(230))) + ) + } + + test("portable SQL line comments end at CR, LF and CRLF") { + Seq("\r", "\n", "\r\n").foreach { + newline => + val expression = RESTApi.toJson("SUM(amount) -- a comment" + newline + " + COUNT(*)") + register(model.replace("sql: SUM(amount)", "sql: " + expression)) + checkAnswer( + spark.sql("SELECT MEASURE(revenue) FROM paimon.sales.order_metrics"), + Seq(Row(BigDecimal(233)))) + } + } + + test("ANSI quoted source columns retain backslashes and embedded double quotes") { + val columns = Seq("tail\\", "back\\\"quote", "apostrophe'name") + def quoted(name: String): String = "`" + name.replace("`", "``") + "`" + val schema = columns.map(name => quoted(name) + " STRING").mkString(", ") + spark.sql( + "CREATE TABLE paimon.sales.quoted_columns (" + schema + + ", amount DECIMAL(12, 2), discount DECIMAL(12, 2)) " + + "TBLPROPERTIES ('bucket'='-1', 'file.format'='parquet')") + spark.sql("INSERT INTO paimon.sales.quoted_columns VALUES ('v0', 'v1', 'v2', 100, 10)") + Seq(false, true).foreach { + escaped => + withSQLConf("spark.sql.parser.escapedStringLiterals" -> escaped.toString) { + columns.zipWithIndex.foreach { + case (column, index) => + val expression = RESTApi.toJson("\"" + column.replace("\"", "\"\"") + "\"") + register( + model + .replace("source: orders", "source: quoted_columns") + .replace("sql: region_code", "sql: " + expression)) + checkAnswer( + spark.sql( + "SELECT region, MEASURE(revenue) FROM paimon.sales.order_metrics GROUP BY region"), + Seq(Row("v" + index, BigDecimal(90)))) + } + } + } + } + + test("all semantic view write targets fail without changing the source") { + register() + val statements = Seq( + "DELETE FROM paimon.sales.order_metrics", + "UPDATE paimon.sales.order_metrics SET region='changed'", + "MERGE INTO paimon.sales.order_metrics m USING (SELECT 'CN' AS region) s " + + "ON m.region=s.region WHEN MATCHED THEN DELETE" + ) + statements.foreach { + statement => + intercept[Exception](spark.sql(statement).collect()) + checkAnswer( + spark.sql("SELECT SUM(amount), COUNT(*) FROM paimon.sales.orders"), + Seq(Row(BigDecimal(280), 3L))) + } + } + + test("preserve REST and source authorization errors") { + register() + Seq(403, 501).foreach { + status => + server.semanticStatus = status + checkAnswer( + spark.sql("SELECT SUM(amount) FROM paimon.sales.orders"), + Seq(Row(BigDecimal(280)))) + val error = intercept[Exception] { + spark.sql("SELECT MEASURE(revenue) FROM paimon.sales.order_metrics").collect() + } + assert(error.toString.contains("semantic access failed")) + } + server.semanticStatus = 200 + server.denySource = true + val error = intercept[Exception] { + spark.sql("SELECT MEASURE(revenue) FROM paimon.sales.order_metrics").collect() + } + assert(error.toString.contains("source access denied")) + server.denySource = false + val missing = intercept[Exception] { + spark.sql("SELECT * FROM paimon.sales.missing_model").collect() + } + assert(missing.toString.contains("TABLE_OR_VIEW_NOT_FOUND")) + } + + test("reject unsupported models, joins, and invalid metric references") { + Seq( + model.replace("version: 0.3.0.dev0", "version: 0.1.1") -> "Invalid Ossie", + model.replace("sql: SUM(amount)", "sql: SUM(discount)") -> "Unknown Ossie dataset field", + model.replace( + "sql: SUM(amount)", + "sql: SUM(orders.amount)") -> "Invalid Ossie metric reference", + model.replace("sql: SUM(amount)", "sql: SUM(gross)") -> "Unknown Ossie dataset field", + model.replace("sql: SUM(amount)", "sql: MEDIAN(amount)") -> "Unsupported OSSIE_SQL_2026", + model.replace("sql: SUM(amount)", "sql: SUM(amount) OVER ()") -> "Unsupported OSSIE_SQL_2026", + model.replace("sql: region_code", "sql: |-\n r'raw'") -> "ANSI string literals", + model.replace("source: orders", "source: other.sales.orders") -> "same Paimon catalog", + model.replace("source: orders", "source: order_metrics") -> "order_metrics" + ).foreach { + case (yaml, message) => + register(yaml) + val error = intercept[Exception] { + spark.sql("SELECT MEASURE(revenue) FROM paimon.sales.order_metrics").collect() + } + assert(error.toString.contains(message), error.toString) + } + register() + val join = intercept[Exception] { + spark + .sql("SELECT MEASURE(m.revenue) FROM paimon.sales.order_metrics m " + + "JOIN paimon.sales.orders o ON m.region = o.region_code GROUP BY m.region") + .collect() + } + assert(join.toString.contains("Joins against Ossie")) + val write = intercept[Exception] { + spark.sql("INSERT INTO paimon.sales.order_metrics SELECT 'CN', 1, 2, 3, 4, 5, 6, 7").collect() + } + assert( + write.toString.contains("read-only") || + write.toString.contains("TABLE_OR_VIEW_NOT_FOUND"), + write.toString) + intercept[Exception] { + spark.readStream.table("paimon.sales.order_metrics") + } + } + + test( + "source identifiers bind to the view catalog and database instead of the current namespace") { + val manager = spark.sessionState.catalogManager + val previous = manager.currentCatalog.name() +: manager.currentNamespace.toSeq + spark.sql("CREATE DATABASE paimon.elsewhere") + spark.sql( + "CREATE TABLE paimon.elsewhere.orders " + + "(region_code STRING, amount DECIMAL(12, 2), discount DECIMAL(12, 2)) " + + "TBLPROPERTIES ('bucket'='-1', 'file.format'='avro')") + spark.sql("INSERT INTO paimon.elsewhere.orders VALUES ('CN', 999, 0)") + try { + spark.sql("USE paimon.elsewhere") + Seq("orders", "sales.orders", "paimon.sales.orders").foreach { + source => + register(model.replace("source: orders", s"source: $source")) + checkAnswer( + spark.sql("SELECT MEASURE(revenue) FROM paimon.sales.order_metrics"), + Seq(Row(BigDecimal(230)))) + } + } finally { + spark.sql("USE " + previous.map(p => "`" + p.replace("`", "``") + "`").mkString(".")) + } + } + + test("analyzed queries retain their definition and queries preserve original metadata") { + val original = model + + "nonportable_extensions:\n - vendor_name: unknown\n data: {date: '2026-11-13', nested: [null, true]}\n" + register(original) + val writes = server.semanticWrites + val retained = spark.sql("SELECT MEASURE(revenue) FROM paimon.sales.order_metrics") + retained.queryExecution.analyzed + checkAnswer(retained, Seq(Row(BigDecimal(230)))) + assert(models.getSemanticView(id).getDefinition.getContent == original) + assert(server.semanticWrites == writes) + register(model.replace("sql: SUM(amount)", "sql: MAX(amount)")) + checkAnswer(retained, Seq(Row(BigDecimal(230)))) + checkAnswer( + spark.sql("SELECT MEASURE(revenue) FROM paimon.sales.order_metrics"), + Seq(Row(BigDecimal(90)))) + } + + private def model: String = + """version: 0.3.0.dev0 + |name: order_metrics + |datasets: + | - name: orders + | source: orders + | fields: + | - name: region + | expression: + | sql: region_code + | - name: amount + | expression: + | sql: amount - discount + | metrics: + | - name: revenue + | expression: + | sql: SUM(amount) + | - name: gross + | expression: + | sql: SUM(source.amount) + | - name: orders + | expression: + | sql: COUNT(*) + | - name: average + | expression: + | sql: AVG(amount) + | - name: distinct_amounts + | expression: + | sql: COUNT(DISTINCT source.amount) + |metrics: + | - name: model_revenue + | expression: + | sql: SUM(orders.amount) + |""".stripMargin +} + +private class SemanticServer(path: String) + extends RESTCatalogServer( + path, + new BearTokenAuthProvider("test-token"), + new ConfigResponse(Collections.singletonMap("prefix", "paimon"), Collections.emptyMap()), + "ossie-tests") { + private val definitions = new ConcurrentHashMap[String, SemanticViewDefinition]() + @volatile var semanticStatus: Int = 200 + @volatile var denySource: Boolean = false + @volatile var authorizedSemanticReads: Int = 0 + @volatile var semanticWrites: Int = 0 + + override def initDispatcher(auth: AuthProvider): Dispatcher = { + val delegate = super.initDispatcher(auth) + new Dispatcher { + override def dispatch(request: RecordedRequest): MockResponse = { + val route = request.getRequestUrl.encodedPath() + def response(status: Int, body: String): MockResponse = new MockResponse() + .setResponseCode(status) + .setHeader("Content-Type", "application/json") + .setBody(body) + if (route.contains("/semantic-views/")) { + val name = request.getRequestUrl + .pathSegments() + .get(request.getRequestUrl.pathSegments().size() - 1) + if (request.getMethod == "POST") { + semanticWrites += 1 + val value = + RESTApi.fromJson(request.getBody.readUtf8(), classOf[UpsertSemanticViewRequest]) + definitions.put(name, value.getDefinition) + } else { + if (semanticStatus != 200) { + return response(semanticStatus, "{\"message\":\"semantic access failed\"}") + } + if (request.getHeader("Authorization") == "Bearer test-token") { + authorizedSemanticReads += 1 + } + } + val definition = definitions.get(name) + if (definition == null) response(404, "{\"message\":\"missing semantic view\"}") + else response(200, RESTApi.toJson(new GetSemanticViewResponse(name, definition))) + } else if (denySource && route.endsWith("/tables/orders")) { + response(403, "{\"message\":\"source access denied\"}") + } else { + delegate.dispatch(request) + } + } + } + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/extensions/PaimonSparkSessionExtensions.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/extensions/PaimonSparkSessionExtensions.scala index e3254f0e347a..22b7eee7e642 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/extensions/PaimonSparkSessionExtensions.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/extensions/PaimonSparkSessionExtensions.scala @@ -41,7 +41,7 @@ class PaimonSparkSessionExtensions extends (SparkSessionExtensions => Unit) { extensions.injectResolutionRule(spark => new PaimonAnalysis(spark)) extensions.injectResolutionRule(spark => PaimonProcedureResolver(spark)) extensions.injectResolutionRule(spark => PaimonViewResolver(spark)) - if (org.apache.spark.SPARK_VERSION.startsWith("4.2.")) { + if (org.apache.spark.SPARK_VERSION >= "4.2") { val resolver = Class .forName("org.apache.spark.sql.paimon.PaimonSemanticViewResolver") .getConstructor(classOf[org.apache.spark.sql.SparkSession]) diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/PaimonV1FunctionRegistry.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/PaimonV1FunctionRegistry.scala index 24bf90c99de5..8fbc5a5c6f8f 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/PaimonV1FunctionRegistry.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/PaimonV1FunctionRegistry.scala @@ -160,13 +160,13 @@ case class PaimonV1FunctionRegistry(session: SparkSession) extends SQLConfHelper } val clazz = PaimonUtils.classForName(className) // The builder name shows up in the default column name of an unaliased call, so leave out - // the catalog that `qualifyIdentifier` keeps on Spark 4.2. + // the catalog that `qualifyIdentifier` keeps on Spark 4.2 and later. val name = func.identifier.copy(catalog = None).unquotedString (input) => functionExpressionBuilder.makeExpression(name, clazz, input) } private def qualifyIdentifier(ident: FunctionIdentifier): FunctionIdentifier = { - val catalog = if (org.apache.spark.SPARK_VERSION.startsWith("4.2.")) { + val catalog = if (org.apache.spark.SPARK_VERSION >= "4.2") { ident.catalog.orElse(Some(session.sessionState.catalogManager.currentCatalog.name)) } else None FunctionIdentifier( diff --git a/paimon-spark/paimon-spark4-common/pom.xml b/paimon-spark/paimon-spark4-common/pom.xml index 032edb98e032..4bbf62d1f14f 100644 --- a/paimon-spark/paimon-spark4-common/pom.xml +++ b/paimon-spark/paimon-spark4-common/pom.xml @@ -153,5 +153,48 @@ under the License. + + spark43 + + + + com.diffplug.spotless + spotless-maven-plugin + + + src/**/*.scala + + + + + net.alchim31.maven + scala-maven-plugin + + + **/Spark4InternalRow.scala + **/Spark4ArrayData.scala + **/Spark41*.scala + **/PaimonSpark4SqlExtensionsParser.scala + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.2.0 + + + spark43-getters + generate-sources + add-source + + src/main/spark43 + + + + + + + diff --git a/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala index 69795c131704..05e96627449e 100644 --- a/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala +++ b/paimon-spark/paimon-spark4-common/src/main/scala/org/apache/spark/sql/paimon/shims/Spark4Shim.scala @@ -186,7 +186,7 @@ class Spark4Shim extends SparkShim { val statementType = "REPLACE TABLE" val columns = schemaOrColumns.asInstanceOf[Seq[ColumnDefinition]] ResolveDefaultColumns.validateCatalogForDefaultValue(columns, catalog, ident) - GeneratedColumn.validateGeneratedColumns(tableSchema, catalog, ident, statementType) + validateGeneratedColumns(tableSchema, columns, catalog, ident, statementType) IdentityColumn.validateIdentityColumn(tableSchema, catalog, ident) columns.map(_.toV2Column(statementType)).toArray } @@ -198,6 +198,27 @@ class Spark4Shim extends SparkShim { tableSpec.copy(properties = tableSpec.properties ++ additionalProperties, location = location) } + // Spark 4.3 keeps only the catalog check here and verifies generation expressions in + // CheckAnalysis. + private def validateGeneratedColumns( + tableSchema: StructType, + columns: Seq[ColumnDefinition], + catalog: TableCatalog, + ident: Identifier, + statementType: String): Unit = { + val methods = GeneratedColumn.getClass.getMethods + val (method, args) = methods.find(_.getName == "validateGeneratedColumns") match { + case Some(m) => (m, Seq(tableSchema, catalog, ident, statementType)) + case None => + val m = methods + .find(_.getName == "validateCatalogForGeneratedColumn") + .getOrElse(throw new IllegalStateException("No generated column validation in Spark")) + (m, Seq(columns, catalog, ident)) + } + try { method.invoke(GeneratedColumn, args: _*) } + catch { case e: java.lang.reflect.InvocationTargetException => throw e.getCause } + } + private def invalidateCache(tableCatalog: TableCatalog, ident: Identifier): Unit = { tableCatalog.invalidateTable(ident) } @@ -405,7 +426,7 @@ class Spark4Shim extends SparkShim { readSpatial(array, pos, geography = true) private def readSpatial(value: AnyRef, pos: Int, geography: Boolean): Array[Byte] = { - val modern = org.apache.spark.SPARK_VERSION.startsWith("4.2.") + val modern = org.apache.spark.SPARK_VERSION >= "4.2" val getter = if (modern) "getBinaryView" else if (geography) "getGeography" else "getGeometry" val raw = value.getClass .getMethod(getter, java.lang.Integer.TYPE) @@ -430,7 +451,7 @@ class Spark4Shim extends SparkShim { override def toSparkGeography(wkb: Array[Byte], crs: String, algorithm: String): Object = { val geographyType = sparkGeographyType(crs, algorithm) val name = - if (org.apache.spark.SPARK_VERSION.startsWith("4.2.")) "stGeogSetSrid" else "stSetSrid" + if (org.apache.spark.SPARK_VERSION >= "4.2") "stGeogSetSrid" else "stSetSrid" val raw = STUtils.stGeogFromWKB(wkb) val method = Class .forName("org.apache.spark.sql.catalyst.util.STUtils") diff --git a/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/catalyst/parser/extensions/Spark43SqlExtensionsParser.scala b/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/catalyst/parser/extensions/Spark43SqlExtensionsParser.scala new file mode 100644 index 000000000000..fb3fa9116896 --- /dev/null +++ b/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/catalyst/parser/extensions/Spark43SqlExtensionsParser.scala @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.catalyst.parser.extensions + +import org.apache.spark.sql.catalyst.parser.{ParserInterface, SqlStatementSplitResult} +import org.apache.spark.sql.catalyst.parser.extensions.AbstractPaimonSparkSqlExtensionsParser +import org.apache.spark.sql.types.StructType + +/** Spark 4.3 adds `splitStatements` to `ParserInterface`. */ +class PaimonSpark4SqlExtensionsParser(override val delegate: ParserInterface) + extends AbstractPaimonSparkSqlExtensionsParser(delegate) + with ParserInterface { + + override def parseRoutineParam(sqlText: String): StructType = delegate.parseRoutineParam(sqlText) + + override def splitStatements(sqlText: String): SqlStatementSplitResult = + delegate.splitStatements(sqlText) +} diff --git a/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/data/Spark43ArrayData.scala b/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/data/Spark43ArrayData.scala new file mode 100644 index 000000000000..d06f843801ba --- /dev/null +++ b/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/data/Spark43ArrayData.scala @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.data + +import org.apache.paimon.types.{DataType, GeographyType, GeometryType} + +import org.apache.spark.sql.paimon.shims.SparkShimLoader +import org.apache.spark.unsafe.types.{BinaryView, TimestampNanosVal, VariantVal} + +/** Spark 4.3 array getters: BinaryView spatial values, as on 4.2, plus nanosecond timestamps. */ +class Spark4ArrayData(override val elementType: DataType) extends AbstractSparkArrayData { + override def getVariant(ordinal: Int): VariantVal = { + val value = paimonArray.getVariant(ordinal) + new VariantVal(value.value(), value.metadata()) + } + + override def getBinaryView(ordinal: Int): BinaryView = { + val bytes = paimonArray.getBinary(ordinal) + val value = elementType match { + case geometry: GeometryType => + SparkShimLoader.shim.toSparkGeometry(bytes, geometry.getCrs) + case geography: GeographyType => + SparkShimLoader.shim.toSparkGeography( + bytes, + geography.getCrs, + geography.getAlgorithm.toString) + case other => throw new UnsupportedOperationException(s"No BinaryView for $other") + } + value.asInstanceOf[BinaryView] + } + + override def getTimestampNTZNanos(ordinal: Int): TimestampNanosVal = + throw new UnsupportedOperationException("No Paimon type maps to Spark nanosecond timestamps") + + override def getTimestampLTZNanos(ordinal: Int): TimestampNanosVal = + throw new UnsupportedOperationException("No Paimon type maps to Spark nanosecond timestamps") +} diff --git a/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/data/Spark43InternalRow.scala b/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/data/Spark43InternalRow.scala new file mode 100644 index 000000000000..398c4caa46be --- /dev/null +++ b/paimon-spark/paimon-spark4-common/src/main/spark43/org/apache/paimon/spark/data/Spark43InternalRow.scala @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.data + +import org.apache.paimon.spark.AbstractSparkInternalRow +import org.apache.paimon.types.{GeographyType, GeometryType, RowType} + +import org.apache.spark.sql.paimon.shims.SparkShimLoader +import org.apache.spark.unsafe.types.{BinaryView, TimestampNanosVal, VariantVal} + +/** Spark 4.3 row getters: BinaryView spatial values, as on 4.2, plus nanosecond timestamps. */ +class Spark4InternalRow(rowType: RowType) extends AbstractSparkInternalRow(rowType) { + override def getVariant(i: Int): VariantVal = { + val value = row.getVariant(i) + new VariantVal(value.value(), value.metadata()) + } + + override def getBinaryView(ordinal: Int): BinaryView = { + val bytes = row.getBinary(ordinal) + val value = rowType.getTypeAt(ordinal) match { + case geometry: GeometryType => + SparkShimLoader.shim.toSparkGeometry(bytes, geometry.getCrs) + case geography: GeographyType => + SparkShimLoader.shim.toSparkGeography( + bytes, + geography.getCrs, + geography.getAlgorithm.toString) + case other => throw new UnsupportedOperationException(s"No BinaryView for $other") + } + value.asInstanceOf[BinaryView] + } + + override def getTimestampNTZNanos(ordinal: Int): TimestampNanosVal = + throw new UnsupportedOperationException("No Paimon type maps to Spark nanosecond timestamps") + + override def getTimestampLTZNanos(ordinal: Int): TimestampNanosVal = + throw new UnsupportedOperationException("No Paimon type maps to Spark nanosecond timestamps") +} diff --git a/pom.xml b/pom.xml index cfde1435edcb..1bf1e625452e 100644 --- a/pom.xml +++ b/pom.xml @@ -475,6 +475,25 @@ under the License. + + spark43 + + paimon-spark/paimon-spark4-common + paimon-spark/paimon-spark-4.3 + + + 17 + 4.13.1 + 2.13 + 2.13.18 + 4.3.0 + paimon-spark4-common_2.13 + 18.1.0 + 4.3 + 4.3.0 + + + flink1 diff --git a/tools/ci/plan.py b/tools/ci/plan.py index e0433ee52e0a..6a01cee53bf1 100644 --- a/tools/ci/plan.py +++ b/tools/ci/plan.py @@ -35,6 +35,7 @@ ("spark3", "Spark 3 / Scala 2.13", 8, "2.13", 90), ("spark4", "Spark 4 / Scala 2.13", 17, "2.13", 90), ("spark42", "Spark 4.2 / Scala 2.13", 17, "2.13", 90), + ("spark43", "Spark 4.3 / Scala 2.13", 17, "2.13", 90), ("e2e-flink1", "E2E / Flink 1 and Spark interoperability", 8, "2.12", 60), ("e2e-flink2", "E2E / Flink 2", 11, "2.12", 60), ("iceberg", "Iceberg GA / JDK 17", 17, "2.12", 60), @@ -46,7 +47,7 @@ REST_SPECS = {"docs/static/rest-catalog-open-api.yaml", "docs/static/rest-management-open-api.yaml"} OPENAPI_V4_ACTIONS = "paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java" FLINK = {"flink1-common", "flink1-connectors", "flink2"} -SPARK = {"spark3", "spark4", "spark42"} +SPARK = {"spark3", "spark4", "spark42", "spark43"} E2E = {"e2e-flink1", "e2e-flink2"} diff --git a/tools/ci/run-java-tests.sh b/tools/ci/run-java-tests.sh index eb8061977fe4..30cea6c63876 100644 --- a/tools/ci/run-java-tests.sh +++ b/tools/ci/run-java-tests.sh @@ -69,6 +69,10 @@ case "$suite" in profiles=flink1,spark42 modules=(org.apache.paimon:paimon-spark-4.2_2.13) ;; + spark43) + profiles=flink1,spark43 + modules=(org.apache.paimon:paimon-spark-4.3_2.13) + ;; e2e-flink1) profiles=flink1,spark3,flink-1.20 modules=(paimon-e2e-tests) diff --git a/tools/ci/test_java_commands.py b/tools/ci/test_java_commands.py index 346ceccc4fef..05d887264a59 100644 --- a/tools/ci/test_java_commands.py +++ b/tools/ci/test_java_commands.py @@ -66,6 +66,9 @@ def test_spark_retains_all_connector_versions_and_scala_variants(self): spark42 = self.commands('spark42', '2.13', 17)[1] self.assertEqual(self.modules(spark42), {'org.apache.paimon:paimon-spark-4.2_2.13'}) self.assertIn('-Pflink1,spark42', spark42) + spark43 = self.commands('spark43', '2.13', 17)[1] + self.assertEqual(self.modules(spark43), {'org.apache.paimon:paimon-spark-4.3_2.13'}) + self.assertIn('-Pflink1,spark43', spark43) def test_core_keeps_common_and_docs_tests_and_existing_jdk11_exclusion(self): for java in (8, 11): diff --git a/tools/ci/test_plan.py b/tools/ci/test_plan.py index 89bcf71c6e0f..58990f360c12 100644 --- a/tools/ci/test_plan.py +++ b/tools/ci/test_plan.py @@ -35,7 +35,7 @@ def suites(self, paths): def test_spark_keeps_common_docs_and_interoperability_tests(self): suites = self.suites(['paimon-spark/paimon-spark-common/src/main/scala/Change.scala']) - self.assertEqual(suites, {'core', 'spark3', 'spark4', 'spark42', 'e2e-flink1'}) + self.assertEqual(suites, {'core', 'spark3', 'spark4', 'spark42', 'spark43', 'e2e-flink1'}) def test_flink_keeps_hive_iceberg_and_both_e2e_versions(self): suites = self.suites(['paimon-flink/paimon-flink-common/src/main/java/Change.java']) @@ -64,7 +64,7 @@ def test_docs_and_markdown_do_not_start_maven(self): def test_mixed_changes_take_union(self): self.assertEqual(self.suites(['docs/intro.md', 'paimon-spark/pom.xml', 'paimon-flink/pom.xml']), - {'core', 'spark3', 'spark4', 'spark42', 'flink1-common', + {'core', 'spark3', 'spark4', 'spark42', 'spark43', 'flink1-common', 'flink1-connectors', 'flink2', 'e2e-flink1', 'e2e-flink2', 'iceberg'}) result = plan.select(['paimon-python/setup.py', 'paimon-core/Change.java']) @@ -220,7 +220,7 @@ def test_pr_size_guard_ignores_unchanged_but_rejects_changed_oversized_files(sel base = self.git('rev-parse', 'HEAD') self.commit('paimon-spark/Change.scala', 'spark') outputs, _ = self.run_pr_plan(base, self.git('rev-parse', 'HEAD')) - self.assertEqual(len(outputs['matrix']['include']), 7) + self.assertEqual(len(outputs['matrix']['include']), 8) for path in ('docs/static/old image.png', 'docs/static/new image.png'): with self.subTest(path=path): @@ -241,7 +241,7 @@ def test_github_outputs_are_valid_json_and_summary_explains_selection(self): outputs = dict(line.split('=', 1) for line in Path('output.txt').read_text().splitlines()) self.assertTrue(json.loads(outputs['java'])) self.assertFalse(json.loads(outputs['python'])) - self.assertEqual(len(json.loads(outputs['matrix'])['include']), 7) + self.assertEqual(len(json.loads(outputs['matrix'])['include']), 8) self.assertIn('Spark interoperability', Path('summary.md').read_text()) From b66bcaab3eac31311ac91038b923c719f3fe12ae Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Thu, 19 Nov 2026 16:00:42 +0800 Subject: [PATCH 2/3] [spark] Bring the Spark 4.3 module up to date with master --- .../sql/paimon/OssieToSparkMetricView.scala | 60 ++++- .../paimon/PaimonSemanticViewResolver.scala | 6 +- .../spark/sql/Spark43CompatibilityTest.scala | 30 ++- .../sql/paimon/OssieSemanticViewTest.scala | 251 +++++++++++++++++- .../extensions/PaimonFunctionLookup.scala | 2 +- 5 files changed, 341 insertions(+), 8 deletions(-) diff --git a/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/OssieToSparkMetricView.scala b/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/OssieToSparkMetricView.scala index 6e5e861a3bb5..441f785915ec 100644 --- a/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/OssieToSparkMetricView.scala +++ b/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/OssieToSparkMetricView.scala @@ -26,22 +26,33 @@ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedFunction, UnresolvedStar} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.parser.CatalystSqlParser +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.trees.Origin +import org.apache.spark.sql.catalyst.util.DateTimeUtils import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.UTF8String +import java.util.Locale + import scala.collection.mutable /** Compile field references before Spark sees the model, preserving source/field shadowing. */ private[paimon] object OssieToSparkMetricView { private val aggregates = Set("sum", "count", "avg", "min", "max") - private val functions = aggregates ++ Set("abs", "round", "floor", "ceil", "ceiling", "coalesce") + private val functions = + aggregates ++ Set("abs", "round", "floor", "ceil", "ceiling", "coalesce", "date_trunc") + private val dateTruncUnits = + Set("YEAR", "QUARTER", "MONTH", "WEEK", "DAY", "HOUR", "MINUTE", "SECOND") + private val dateTruncFunction = Seq("system", "builtin", "date_trunc") + + case class Conversion(yaml: String, dateTruncOrigins: Set[Origin]) def convert( spark: SparkSession, model: OssieModel, source: Seq[String], - sourceColumns: Seq[String]): String = { + sourceColumns: Seq[String]): Conversion = { val sourceNames = sourceColumns.map(n => OssieModel.normalize(n) -> n).toMap require(sourceNames.size == sourceColumns.size, "Ambiguous case-insensitive source columns") val publicNames = @@ -115,8 +126,30 @@ private[paimon] object OssieToSparkMetricView { measures.foreach { case (name, expr) => metricNodes.addObject().put("name", name).put("expr", expr) } + val parser = spark.sessionState.sqlParser + val expressions = parser.parsePlan(sql).expressions ++ + measures.map { case (_, expression) => parser.parseExpression(expression) } + val dateTruncOrigins = expressions + .flatMap(_.collect { + case f: UnresolvedFunction if f.nameParts == dateTruncFunction => f.origin + }) + .toSet // JSON is a YAML subset, and avoids another serialization path for SQL strings. - mapper.writeValueAsString(root) + Conversion(mapper.writeValueAsString(root), dateTruncOrigins) + } + + def validateDateTruncInputs(plan: LogicalPlan, dateTruncOrigins: Set[Origin]): Unit = { + // Spark inserts the session-zone cast during analysis, including for computed arguments. + // Match model call origins so SQL UDF bodies retain their own timestamp semantics. + val truncatesLocalTimestamp = plan.exists { + node => + node.expressions.exists(_.exists { + case t @ TruncTimestamp(_, cast: Cast, _) => + dateTruncOrigins.contains(t.origin) && cast.child.dataType == TimestampNTZType + case _ => false + }) + } + require(!truncatesLocalTimestamp, "Ossie DATE_TRUNC does not support TIMESTAMP_NTZ inputs") } private def parse(spark: SparkSession, sql: String, metric: Boolean): Expression = { @@ -138,6 +171,17 @@ private[paimon] object OssieToSparkMetricView { s"Unsupported OSSIE_SQL_2026 function: ${f.nameParts.mkString(".")}" ) val name = OssieModel.normalize(f.nameParts.head) + if (name == "date_trunc") { + f.arguments match { + case Seq(Literal(unit: UTF8String, _: StringType), _) => + require( + dateTruncUnits.contains(unit.toString.toUpperCase(Locale.ROOT)), + s"Unsupported Ossie DATE_TRUNC unit: $unit") + case _ => + throw new IllegalArgumentException( + "Ossie DATE_TRUNC expects a unit string literal followed by an input expression") + } + } if (aggregates.contains(name)) { require(metric, "Ossie field expressions cannot contain aggregates") aggregate = true @@ -238,11 +282,19 @@ private[paimon] object OssieToSparkMetricView { // Native Metric View parsing uses session settings; binary literals avoid a second round // of string escaping regardless of escapedStringLiterals. RenderedSQL(s"CAST(X'$hex' AS STRING)") + case Literal(micros: Long, TimestampType) => + // Use an explicit type and UTC offset so reparsing preserves the instant. + RenderedSQL(s"TIMESTAMP_LTZ '${DateTimeUtils.microsToInstant(micros)}'") case f: UnresolvedFunction => + val nameParts = if (OssieModel.normalize(f.nameParts.head) == "date_trunc") { + dateTruncFunction + } else { + f.nameParts + } val distinct = if (f.isDistinct) "DISTINCT " else "" val filter = f.filter.map(p => s" FILTER (WHERE ${p.sql})").getOrElse("") RenderedSQL( - s"${f.nameParts.map(quote).mkString(".")}($distinct${f.arguments.map(_.sql).mkString(", ")})$filter") + s"${nameParts.map(quote).mkString(".")}($distinct${f.arguments.map(_.sql).mkString(", ")})$filter") }.sql private case class RenderedSQL(text: String) extends LeafExpression with Unevaluable { diff --git a/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/PaimonSemanticViewResolver.scala b/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/PaimonSemanticViewResolver.scala index d3a0ca25ecc9..59cd13608f30 100644 --- a/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/PaimonSemanticViewResolver.scala +++ b/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/PaimonSemanticViewResolver.scala @@ -94,12 +94,16 @@ class PaimonSemanticViewResolver(spark: SparkSession) require( table.isInstanceOf[FileStoreTable], "Ossie source must be a physical Paimon table") - val yaml = OssieToSparkMetricView.convert( + val conversion = OssieToSparkMetricView.convert( spark, model, source, table.rowType().getFieldNames.asScala.toSeq) + val yaml = conversion.yaml val (analyzed, _) = MetricViewHelper.analyzeMetricViewText(spark, parts, yaml) + OssieToSparkMetricView.validateDateTruncInputs( + analyzed, + conversion.dateTruncOrigins) val metadata = CatalogTable( identifier = TableIdentifier( ident.name(), diff --git a/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/sql/Spark43CompatibilityTest.scala b/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/sql/Spark43CompatibilityTest.scala index 0fb254110917..82bd51ce35a7 100644 --- a/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/sql/Spark43CompatibilityTest.scala +++ b/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/sql/Spark43CompatibilityTest.scala @@ -26,7 +26,7 @@ import org.apache.paimon.spark.catalyst.parser.extensions.PaimonSpark4SqlExtensi import org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions import org.apache.spark.SparkConf -import org.apache.spark.sql.{QueryTest, Row} +import org.apache.spark.sql.{AnalysisException, QueryTest, Row} import org.apache.spark.sql.test.SharedSparkSession import java.nio.file.Files @@ -110,6 +110,20 @@ class Spark43CompatibilityTest extends QueryTest with SharedSparkSession { } } + test("replace table rejects table constraints and keeps the old table") { + withTable("t") { + sql("CREATE TABLE t (id INT) TBLPROPERTIES ('file.format'='avro')") + sql("INSERT INTO t VALUES (7)") + val e = intercept[AnalysisException] { + sql( + "CREATE OR REPLACE TABLE t (id INT, CONSTRAINT positive CHECK (id > 0)) " + + "TBLPROPERTIES ('file.format'='avro')") + } + assert(e.getCondition === "UNSUPPORTED_FEATURE.TABLE_OPERATION") + checkAnswer(sql("SELECT * FROM t"), Seq(Row(7))) + } + } + test("split statements through the Paimon parser") { val parser = spark.sessionState.sqlParser assert(parser.isInstanceOf[PaimonSpark4SqlExtensionsParser]) @@ -227,4 +241,18 @@ class Spark43CompatibilityTest extends QueryTest with SharedSparkSession { ) } } + + test("spatial SQL passes a constant geometry to the Paimon scan") { + val point = + "ST_SetSrid(ST_GeomFromWKB(unhex('01010000000000000000000000000000000000F03F')), 4326)" + withTable("locations") { + sql("""CREATE TABLE locations (geom GEOMETRY(4326)) TBLPROPERTIES ( + | 'bucket' = '-1', 'row-tracking.enabled' = 'true', + | 'data-evolution.enabled' = 'true')""".stripMargin) + sql(s"INSERT INTO locations VALUES ($point), (NULL)") + val query = sql(s"SELECT ST_Srid(geom) FROM locations WHERE ST_Intersects(geom, $point)") + assert(query.queryExecution.executedPlan.toString().contains("BBoxIntersects")) + checkAnswer(query, Seq(Row(4326))) + } + } } diff --git a/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala b/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala index 42ea9561e3e1..80fd87caa11f 100644 --- a/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala +++ b/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala @@ -32,9 +32,11 @@ import okhttp3.mockwebserver.{Dispatcher, MockResponse, RecordedRequest} import org.apache.spark.SparkConf import org.apache.spark.sql.{QueryTest, Row} import org.apache.spark.sql.test.SharedSparkSession +import org.apache.spark.sql.types.TimestampType import java.nio.file.Files -import java.util.Collections +import java.time.Instant +import java.util.{Collections, Locale} import java.util.concurrent.ConcurrentHashMap /** Real Spark queries, real REST client, and physical Paimon source files. */ @@ -119,6 +121,253 @@ class OssieSemanticViewTest extends QueryTest with SharedSparkSession { Seq(Row(null, 0L))) } + test("group Ossie measures by DATE_TRUNC and preserve Spark timestamp semantics") { + withSQLConf("spark.sql.session.timeZone" -> "UTC") { + spark.sql( + "CREATE TABLE paimon.sales.date_trunc_orders " + + "(order_date DATE, order_time TIMESTAMP_LTZ, amount BIGINT) " + + "TBLPROPERTIES ('bucket' = '-1')") + spark.sql("""INSERT INTO paimon.sales.date_trunc_orders VALUES + | (DATE '2026-05-10', NULL, 10), + | (DATE '2026-05-31', TIMESTAMP '2026-05-31 13:42:18.123456', 20), + | (DATE '2026-06-01', TIMESTAMP '2026-06-01 00:00:00.987654', 7), + | (NULL, NULL, 3)""".stripMargin) + + val units = Seq("year", "quarter", "month", "week", "day", "hour", "minute", "second") + val timestampFields = units + .map { + unit => + s""" - name: by_$unit + | expression: + | sql: date_trunc('${unit.toUpperCase(Locale.ROOT)}', order_time)""".stripMargin + } + .mkString("\n") + register(s"""version: 0.3.0.dev0 + |name: order_metrics + |datasets: + | - name: orders + | source: date_trunc_orders + | fields: + | - name: order_month + | expression: + | sql: DATE_TRUNC('month', order_date) + |$timestampFields + | metrics: + | - name: revenue + | expression: + | sql: SUM(source.amount) + |""".stripMargin) + + val monthly = spark.sql( + "SELECT order_month, MEASURE(revenue) AS revenue " + + "FROM paimon.sales.order_metrics GROUP BY order_month") + assert(monthly.schema("order_month").dataType == TimestampType) + checkAnswer( + monthly.selectExpr("CAST(order_month AS STRING)", "revenue"), + Seq(Row("2026-05-01 00:00:00", 30L), Row("2026-06-01 00:00:00", 7L), Row(null, 3L))) + + val columns = units.map(unit => s"by_$unit") + val grouping = columns.mkString(", ") + val timestamps = spark.sql( + s"SELECT $grouping, MEASURE(revenue) AS revenue " + + s"FROM paimon.sales.order_metrics GROUP BY $grouping") + assert(columns.forall(name => timestamps.schema(name).dataType == TimestampType)) + val formatted = columns.map(name => s"CAST($name AS STRING)") :+ "revenue" + checkAnswer( + timestamps.selectExpr(formatted: _*), + Seq( + Row( + "2026-01-01 00:00:00", + "2026-04-01 00:00:00", + "2026-05-01 00:00:00", + "2026-05-25 00:00:00", + "2026-05-31 00:00:00", + "2026-05-31 13:00:00", + "2026-05-31 13:42:00", + "2026-05-31 13:42:18", + 20L + ), + Row( + "2026-01-01 00:00:00", + "2026-04-01 00:00:00", + "2026-06-01 00:00:00", + "2026-06-01 00:00:00", + "2026-06-01 00:00:00", + "2026-06-01 00:00:00", + "2026-06-01 00:00:00", + "2026-06-01 00:00:00", + 7L + ), + Row.fromSeq(Seq.fill(columns.size)(null) :+ 13L) + ) + ) + } + } + + test("reject TIMESTAMP_NTZ inputs to Ossie DATE_TRUNC during analysis") { + spark.sql( + "CREATE TABLE paimon.sales.local_orders (order_time TIMESTAMP_NTZ, amount BIGINT) " + + "TBLPROPERTIES ('bucket' = '-1')") + spark.sql("""INSERT INTO paimon.sales.local_orders VALUES + | (TIMESTAMP_NTZ '2026-03-08 02:30:00', 10), + | (TIMESTAMP_NTZ '2026-03-08 03:30:00', 20)""".stripMargin) + val yaml = """version: 0.3.0.dev0 + |name: order_metrics + |datasets: + | - name: orders + | source: local_orders + | fields: + | - name: order_hour + | expression: + | sql: DATE_TRUNC('hour', order_time) + | metrics: + | - name: revenue + | expression: + | sql: SUM(source.amount) + |""".stripMargin + register(yaml) + val query = "SELECT order_hour, MEASURE(revenue) AS revenue " + + "FROM paimon.sales.order_metrics GROUP BY order_hour" + val message = "Ossie DATE_TRUNC does not support TIMESTAMP_NTZ inputs" + Seq("UTC", "America/Los_Angeles").foreach { + zone => + withSQLConf("spark.sql.session.timeZone" -> zone) { + val error = intercept[IllegalArgumentException] { + spark.sql(query).schema + } + assert(error.getMessage.contains(message), error.toString) + } + } + + register(yaml + .replace("DATE_TRUNC('hour', order_time)", "order_time") + .replace("SUM(source.amount)", "MAX(DATE_TRUNC('hour', COALESCE(source.order_time, NULL)))")) + val error = intercept[IllegalArgumentException] { + spark.sql("SELECT MEASURE(revenue) FROM paimon.sales.order_metrics").schema + } + assert(error.getMessage.contains(message), error.toString) + } + + test("accept Spark timestamp strings in Ossie DATE_TRUNC") { + withSQLConf("spark.sql.session.timeZone" -> "UTC") { + register(model.replace("sql: region_code", "sql: DATE_TRUNC('month', '2026-06-01')")) + val result = spark.sql( + "SELECT region, MEASURE(gross) AS gross FROM paimon.sales.order_metrics GROUP BY region") + assert(result.schema("region").dataType == TimestampType) + checkAnswer( + result.selectExpr("CAST(region AS STRING)", "gross"), + Row("2026-06-01 00:00:00", BigDecimal(280))) + } + } + + test("DATE_TRUNC input validation preserves SQL UDF body semantics") { + val session = spark.newSession() + session.conf.set("spark.sql.path.enabled", "true") + session.conf.set("spark.sql.session.timeZone", "UTC") + session.sql( + "CREATE FUNCTION spark_catalog.default.floor(ts TIMESTAMP_NTZ) " + + "RETURNS TIMESTAMP_LTZ RETURN date_trunc('day', ts)") + try { + session.sql("SET PATH = spark_catalog.default, system.builtin") + val floor = "FLOOR(TIMESTAMP_NTZ '2026-06-12 10:11:12')" + val truncate = "DATE_TRUNC('day', TIMESTAMP_LTZ '2026-06-12 10:11:12Z')" + Seq(floor, s"COALESCE($floor, $truncate)", s"DATE_TRUNC('day', $floor)").foreach { + expression => + register(model.replace("sql: region_code", s"sql: $expression")) + val result = session.sql( + "SELECT region, MEASURE(gross) AS gross FROM paimon.sales.order_metrics GROUP BY region") + assert(result.schema("region").dataType == TimestampType) + checkAnswer( + result.selectExpr("CAST(region AS STRING)", "gross"), + Row("2026-06-12 00:00:00", BigDecimal(280))) + } + register(model.replace("sql: SUM(amount)", s"sql: MAX($floor)")) + checkAnswer( + session.sql("SELECT CAST(MEASURE(revenue) AS STRING) FROM paimon.sales.order_metrics"), + Row("2026-06-12 00:00:00")) + } finally { + session.sql("DROP FUNCTION spark_catalog.default.floor") + } + } + + test("resolve Ossie DATE_TRUNC to the Spark builtin regardless of the SQL path") { + val session = spark.newSession() + session.conf.set("spark.sql.path.enabled", "true") + session.conf.set("spark.sql.session.timeZone", "UTC") + session.sql( + "CREATE FUNCTION spark_catalog.default.date_trunc(unit STRING, ts TIMESTAMP_LTZ) " + + "RETURNS STRING RETURN 'shadow'") + try { + session.sql("SET PATH = spark_catalog.default, system.builtin") + val timestamp = "TIMESTAMP_LTZ '2026-06-12 10:11:12Z'" + checkAnswer(session.sql(s"SELECT date_trunc('month', $timestamp)"), Row("shadow")) + + register(model.replace("sql: region_code", s"sql: DaTe_TrUnC('month', $timestamp)")) + val query = + "SELECT region, MEASURE(gross) AS gross FROM paimon.sales.order_metrics GROUP BY region" + val result = session.sql(query) + checkAnswer( + result.selectExpr("CAST(region AS STRING)", "gross"), + Row("2026-06-01 00:00:00", BigDecimal(280))) + assert(result.schema("region").dataType == TimestampType) + + register( + model.replace( + "sql: region_code", + "sql: DATE_TRUNC('month', TIMESTAMP_NTZ '2026-06-12 10:11:12')")) + val error = intercept[IllegalArgumentException] { + session.sql(query).schema + } + assert(error.getMessage.contains("Ossie DATE_TRUNC does not support TIMESTAMP_NTZ inputs")) + } finally { + session.sql("DROP FUNCTION spark_catalog.default.date_trunc") + } + } + + Seq("TIMESTAMP_LTZ", "TIMESTAMP_NTZ").foreach { + timestampType => + test(s"preserve Ossie DATE_TRUNC literal instants with $timestampType as the default") { + withSQLConf( + "spark.sql.timestampType" -> timestampType, + "spark.sql.session.timeZone" -> "America/Los_Angeles") { + val timestamp = "CASE WHEN region_code = 'CN' " + + "THEN TIMESTAMP_LTZ '2026-11-01 01:35:00.123456-07:00' " + + "ELSE TIMESTAMP_LTZ '2026-11-01 01:35:00.654321-08:00' END" + register(model.replace("sql: region_code", s"sql: DATE_TRUNC('hour', $timestamp)")) + val result = spark.sql( + "SELECT region, MEASURE(gross) AS gross FROM paimon.sales.order_metrics GROUP BY region") + assert(result.schema("region").dataType == TimestampType) + checkAnswer( + result.selectExpr("unix_micros(region)", "gross"), + Seq( + Row(Instant.parse("2026-11-01T08:00:00Z").getEpochSecond * 1000000L, BigDecimal(200)), + Row(Instant.parse("2026-11-01T09:00:00Z").getEpochSecond * 1000000L, BigDecimal(80)) + ) + ) + } + } + } + + test("reject invalid DATE_TRUNC units and arguments in Ossie models") { + val date = "DATE '2026-06-01'" + Seq( + s"DATE_TRUNC('mont', $date)" -> "Unsupported Ossie DATE_TRUNC unit: mont", + s"DATE_TRUNC('MM', $date)" -> "Unsupported Ossie DATE_TRUNC unit: MM", + s"DATE_TRUNC('millisecond', $date)" -> "Unsupported Ossie DATE_TRUNC unit: millisecond", + s"DATE_TRUNC('WEE\u212A', $date)" -> "Unsupported Ossie DATE_TRUNC unit: WEE\u212A", + s"DATE_TRUNC(region_code, $date)" -> "Ossie DATE_TRUNC expects a unit string literal", + s"DATE_TRUNC(NULL, $date)" -> "Ossie DATE_TRUNC expects a unit string literal", + "DATE_TRUNC('month')" -> "Ossie DATE_TRUNC expects a unit string literal" + ).foreach { + case (expression, message) => + register(model.replace("sql: region_code", s"sql: $expression")) + val error = intercept[IllegalArgumentException] { + spark.sql("SELECT region FROM paimon.sales.order_metrics").collect() + } + assert(error.getMessage.contains(message), error.toString) + } + } + test("public field names cannot shadow private metric input aliases") { Seq("__ossie_f1", "__ossie_0_f1").foreach { name => diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonFunctionLookup.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonFunctionLookup.scala index 7f530d7912ef..b23d15df293f 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonFunctionLookup.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/PaimonFunctionLookup.scala @@ -54,7 +54,7 @@ case class PaimonFunctionLookup(catalogManager: CatalogManager) extends LookupCa Some(null, FunctionIdentifier(funName), true) // Spark 4.2 keeps extension functions in the session's builtin namespace. case Seq(name) - if org.apache.spark.SPARK_VERSION.startsWith("4.2.") && + if org.apache.spark.SPARK_VERSION >= "4.2" && catalogManager.v1SessionCatalog.lookupBuiltinOrTempFunction(name).isDefined => None case CatalogAndIdentifier(v1FunctionCatalog: SupportV1Function, ident) From 28a7ef8f44793fa3038956ca4cc0db62bf826bf8 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Fri, 20 Nov 2026 11:39:47 +0800 Subject: [PATCH 3/3] [spark] Compile the Spark 4.2 module sources in the Spark 4.3 module --- .../spark/sql/Spark42CompatibilityTest.scala | 28 +- .../sql/paimon/OssieSemanticViewTest.scala | 2 + paimon-spark/paimon-spark-4.3/pom.xml | 48 ++ .../paimon/spark/ossie/ossie-schema.json | 396 ---------- .../paimon/spark/ossie/OssieModel.scala | 152 ---- .../Spark42RowLevelCommandRewrite.scala | 65 -- .../sql/paimon/OssieToSparkMetricView.scala | 305 ------- .../paimon/PaimonSemanticViewResolver.scala | 150 ---- .../data/SparkInternalRowVariantTest.scala | 54 -- .../paimon/spark/ossie/OssieModelTest.scala | 136 ---- .../spark/sql/Spark43CompatibilityTest.scala | 221 +----- .../sql/paimon/OssieSemanticViewTest.scala | 746 ------------------ 12 files changed, 79 insertions(+), 2224 deletions(-) delete mode 100644 paimon-spark/paimon-spark-4.3/src/main/resources/org/apache/paimon/spark/ossie/ossie-schema.json delete mode 100644 paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/paimon/spark/ossie/OssieModel.scala delete mode 100644 paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/catalyst/analysis/Spark42RowLevelCommandRewrite.scala delete mode 100644 paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/OssieToSparkMetricView.scala delete mode 100644 paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/PaimonSemanticViewResolver.scala delete mode 100644 paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/data/SparkInternalRowVariantTest.scala delete mode 100644 paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/ossie/OssieModelTest.scala delete mode 100644 paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala diff --git a/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/Spark42CompatibilityTest.scala b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/Spark42CompatibilityTest.scala index d535c2f6c275..45d21c3168aa 100644 --- a/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/Spark42CompatibilityTest.scala +++ b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/paimon/spark/sql/Spark42CompatibilityTest.scala @@ -31,7 +31,10 @@ import org.apache.spark.sql.test.SharedSparkSession import java.nio.file.Files import java.util.Collections -/** Exercise the APIs whose signatures changed in Spark 4.2 using real Paimon files. */ +/** + * Exercise the APIs whose signatures changed in Spark 4.2 using real Paimon files. The Spark 4.3 + * connector compiles and runs this suite too. + */ class Spark42CompatibilityTest extends QueryTest with SharedSparkSession { private val warehouse = Files.createTempDirectory("paimon-spark42-").toFile private lazy val server = { @@ -54,6 +57,8 @@ class Spark42CompatibilityTest extends QueryTest with SharedSparkSession { .set("spark.sql.catalog.paimon.token", "test-token") .set("spark.sql.defaultCatalog", "paimon") .set("spark.sql.extensions", classOf[PaimonSparkSessionExtensions].getName) + // Spark 4.3 dual-runs the single-pass resolver in tests, and it cannot resolve Paimon views. + .set("spark.sql.analyzer.singlePassResolver.dualRunWithLegacy", "false") override protected def beforeAll(): Unit = { super.beforeAll() @@ -86,6 +91,27 @@ class Spark42CompatibilityTest extends QueryTest with SharedSparkSession { } } + test("create or replace table as select") { + withTable("t") { + sql("CREATE TABLE t (id INT, label STRING) TBLPROPERTIES ('file.format'='avro')") + sql("INSERT INTO t VALUES (1, 'old')") + sql( + "CREATE OR REPLACE TABLE t TBLPROPERTIES ('file.format'='avro') " + + "AS SELECT 2 AS id, 'new' AS label") + checkAnswer(sql("SELECT * FROM t"), Seq(Row(2, "new"))) + } + } + + test("replace table with a column list") { + withTable("t") { + sql("CREATE TABLE t (id INT, label STRING) TBLPROPERTIES ('file.format'='avro')") + sql("INSERT INTO t VALUES (1, 'old')") + sql("CREATE OR REPLACE TABLE t (id INT, score DOUBLE) TBLPROPERTIES ('file.format'='avro')") + sql("INSERT INTO t VALUES (2, 0.5)") + checkAnswer(sql("SELECT * FROM t"), Seq(Row(2, 0.5))) + } + } + test("replace table rejects table constraints and keeps the old table") { withTable("t") { sql("CREATE TABLE t (id INT) TBLPROPERTIES ('file.format'='avro')") diff --git a/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala index f783e4a47014..80fd87caa11f 100644 --- a/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala +++ b/paimon-spark/paimon-spark-4.2/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala @@ -65,6 +65,8 @@ class OssieSemanticViewTest extends QueryTest with SharedSparkSession { .set("spark.sql.catalog.paimon.token", "test-token") .set("spark.sql.catalog.paimon.cache-enabled", "false") .set("spark.sql.extensions", classOf[PaimonSparkSessionExtensions].getName) + // Spark 4.3 dual-runs the single-pass resolver in tests, and it cannot resolve Paimon views. + .set("spark.sql.analyzer.singlePassResolver.dualRunWithLegacy", "false") override protected def beforeAll(): Unit = { super.beforeAll() diff --git a/paimon-spark/paimon-spark-4.3/pom.xml b/paimon-spark/paimon-spark-4.3/pom.xml index b0bcb9afa540..201e3d3e31f6 100644 --- a/paimon-spark/paimon-spark-4.3/pom.xml +++ b/paimon-spark/paimon-spark-4.3/pom.xml @@ -31,10 +31,15 @@ under the License. paimon-spark-4.3_2.13 Paimon : Spark : 4.3 : 2.13 + 4.3.0 2.13 2.13.18 + ${project.basedir}/../paimon-spark-4.2 @@ -133,7 +138,50 @@ under the License. + + + src/main/resources + + + ${spark42.module.dir}/src/main/resources + + META-INF/NOTICE + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-spark42-sources + generate-sources + + add-source + + + + ${spark42.module.dir}/src/main/scala + + + + + add-spark42-test-sources + generate-test-sources + + add-test-source + + + + ${spark42.module.dir}/src/test/scala + + + + + + org.apache.maven.plugins maven-shade-plugin diff --git a/paimon-spark/paimon-spark-4.3/src/main/resources/org/apache/paimon/spark/ossie/ossie-schema.json b/paimon-spark/paimon-spark-4.3/src/main/resources/org/apache/paimon/spark/ossie/ossie-schema.json deleted file mode 100644 index ac15741121b8..000000000000 --- a/paimon-spark/paimon-spark-4.3/src/main/resources/org/apache/paimon/spark/ossie/ossie-schema.json +++ /dev/null @@ -1,396 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/apache/ossie/core-spec/ossie-schema.json", - "title": "Apache Ossie Core Metadata Specification", - "description": "JSON Schema for validating a single Apache Ossie semantic model document", - "type": "object", - "properties": { - "version": { - "type": "string", - "const": "0.3.0.dev0", - "description": "Apache Ossie specification version" - }, - "name": { - "$ref": "#/$defs/SemanticModel/properties/name" - }, - "description": { - "$ref": "#/$defs/SemanticModel/properties/description" - }, - "ai_context": { - "$ref": "#/$defs/SemanticModel/properties/ai_context" - }, - "datasets": { - "$ref": "#/$defs/SemanticModel/properties/datasets" - }, - "relationships": { - "$ref": "#/$defs/SemanticModel/properties/relationships" - }, - "metrics": { - "$ref": "#/$defs/SemanticModel/properties/metrics" - }, - "nonportable_extensions": { - "$ref": "#/$defs/SemanticModel/properties/nonportable_extensions" - } - }, - "required": ["version", "name", "datasets"], - "additionalProperties": false, - "$defs": { - "Dialect": { - "type": "string", - "minLength": 1, - "examples": ["OSSIE_SQL_2026", "ANSI_SQL", "DB_DIALECT", "BI_EXPRESSION", "SEMANTIC_MODEL_EXPRESSION", "PROGRAMMING_LANGUAGE"], - "description": "Expression language a metric or field expression is declared in. Any non-empty string is accepted. OSSIE_SQL_2026 is the only dialect this specification defines; others are opaque to Ossie. See extensions/dialects.md." - }, - "Vendor": { - "type": "string", - "minLength": 1, - "examples": ["BI_TOOL", "DB_VENDOR", "SEMANTIC_LAYER"], - "description": "Vendor name for non-portable extensions. Any non-empty string is accepted. See extensions/vendors.md." - }, - "AIContext": { - "description": "Additional context for AI tools", - "oneOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "instructions": { - "type": "string", - "description": "Instructions for AI on how to use this entity" - }, - "synonyms": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Alternative names and terms" - }, - "examples": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Sample questions or use cases" - } - }, - "additionalProperties": true - } - ] - }, - "NonportableExtension": { - "type": "object", - "description": "Vendor-specific state that Ossie carries and does not interpret. Portable behavior does not depend on it.", - "properties": { - "vendor_name": { - "$ref": "#/$defs/Vendor" - }, - "data": { - "type": "object", - "description": "Vendor-specific data as a structured object. The core schema does not reference the structure under data: no reserved key and no common sub-key. Keys are entirely vendor-defined. Values must be JSON-representable. Ossie carries the object but does not interpret it. See core-spec/spec.md#non-portable-extensions." - } - }, - "required": ["vendor_name", "data"], - "additionalProperties": false - }, - "DialectExpression": { - "type": "object", - "description": "Expression in a specific dialect", - "properties": { - "dialect": { - "$ref": "#/$defs/Dialect" - }, - "expression": { - "type": "string", - "minLength": 1, - "description": "SQL or dialect-specific expression" - } - }, - "required": ["dialect", "expression"], - "additionalProperties": false - }, - "Expression": { - "type": "object", - "description": "Expression definition with multi-dialect support", - "properties": { - "sql": { - "type": "string", - "minLength": 1, - "description": "Primary, portable expression written in OSSIE_SQL_2026. When both this field and an OSSIE_SQL_2026 entry in `dialects` are present, this field is authoritative." - }, - "dialects": { - "type": "array", - "items": { - "$ref": "#/$defs/DialectExpression" - }, - "minItems": 1 - } - }, - "anyOf": [ - { "required": ["sql"] }, - { "required": ["dialects"] } - ], - "additionalProperties": false - }, - "DataType": { - "type": "string", - "enum": [ - "String", - "Integer", - "Decimal", - "Float", - "Boolean", - "Date", - "Time", - "DateTime", - "DateTimeTz", - "Opaque" - ], - "description": "Logical data type for fields and metrics, independent of role (e.g. dimension vs fact) and physical representation. `Decimal` is exact base-10 with unspecified precision and scale; `Float` is approximate. `DateTime` has no timezone or offset, while `DateTimeTz` identifies an instant using offset or timezone context but does not guarantee preservation of a named timezone. Omit `datatype` when unknown. `Opaque` means the type is outside the portable vocabulary. A vendor may record its own type name under `nonportable_extensions`; that record is not part of the portable type." - }, - "Dimension": { - "type": "object", - "description": "Dimension metadata", - "properties": { - "is_time": { - "type": "boolean", - "description": "Temporal-role marker. When true, consumers that distinguish time dimensions (e.g. for time-series analysis or temporal filtering) should treat this field as a time dimension. This is a *role* flag, independent of the field's data type: a field with `is_time: true` may carry any `datatype` (e.g. `Integer` for a year grain, `String` for a month name, as well as temporal data types). When `is_time` is unset, it defaults to `true` if `datatype` is one of `Date`, `Time`, `DateTime`, or `DateTimeTz`, and `false` otherwise. Set `is_time: false` explicitly to opt a temporal-typed column (such as an audit timestamp) out of time-dimension treatment." - } - }, - "additionalProperties": false - }, - "Field": { - "type": "object", - "description": "Row-level attribute for grouping, filtering, and metric expressions", - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Unique identifier for the field within the dataset" - }, - "expression": { - "$ref": "#/$defs/Expression" - }, - "dimension": { - "$ref": "#/$defs/Dimension" - }, - "label": { - "type": "string", - "description": "Label for categorization" - }, - "description": { - "type": "string", - "description": "Human-readable description" - }, - "datatype": { - "$ref": "#/$defs/DataType" - }, - "ai_context": { - "$ref": "#/$defs/AIContext" - }, - "nonportable_extensions": { - "type": "array", - "items": { - "$ref": "#/$defs/NonportableExtension" - } - } - }, - "required": ["name", "expression"], - "additionalProperties": false - }, - "Dataset": { - "type": "object", - "description": "Logical dataset representing a business entity (fact or dimension table)", - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Unique identifier for the dataset" - }, - "source": { - "type": "string", - "minLength": 1, - "description": "Reference to underlying physical table/view (database.schema.table) or query" - }, - "primary_key": { - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1, - "description": "Dataset fields that uniquely identify rows" - }, - "unique_keys": { - "type": "array", - "items": { - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1 - }, - "minItems": 1, - "description": "Array of unique key definitions, each a list of dataset fields" - }, - "description": { - "type": "string", - "description": "Human-readable description" - }, - "ai_context": { - "$ref": "#/$defs/AIContext" - }, - "fields": { - "type": "array", - "items": { - "$ref": "#/$defs/Field" - }, - "description": "The dataset's declared fields. Within the dataset, field names and metric names share one namespace and must all be unique across the two." - }, - "metrics": { - "type": "array", - "items": { - "$ref": "#/$defs/Metric" - }, - "description": "Dataset-scoped metrics: aggregations evaluated at this dataset's row level, for measures that semantically belong to the dataset. The expression references this dataset's declared fields by single-part (unqualified) name, and columns of the dataset's source as source.column. It must resolve entirely within this dataset; an expression that must reference another dataset belongs in the model-scoped metrics instead. Within the dataset, field names and metric names share one namespace and must all be unique across the two." - }, - "nonportable_extensions": { - "type": "array", - "items": { - "$ref": "#/$defs/NonportableExtension" - } - } - }, - "required": ["name", "source"], - "additionalProperties": false - }, - "Relationship": { - "type": "object", - "description": "Foreign key relationship between datasets", - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Unique identifier for the relationship" - }, - "from": { - "type": "string", - "minLength": 1, - "description": "Dataset on the many side of the relationship" - }, - "to": { - "type": "string", - "minLength": 1, - "description": "Dataset on the one side of the relationship" - }, - "from_columns": { - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1, - "description": "Array of field names in the 'from' dataset (foreign key fields)" - }, - "to_columns": { - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1, - "description": "Array of field names in the 'to' dataset (primary or unique key fields)" - }, - "ai_context": { - "$ref": "#/$defs/AIContext" - }, - "nonportable_extensions": { - "type": "array", - "items": { - "$ref": "#/$defs/NonportableExtension" - } - } - }, - "required": ["name", "from", "to", "from_columns", "to_columns"], - "additionalProperties": false - }, - "Metric": { - "type": "object", - "description": "Quantitative measure defined on business data", - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Unique identifier for the metric" - }, - "expression": { - "$ref": "#/$defs/Expression" - }, - "description": { - "type": "string", - "description": "Human-readable description of what the metric measures" - }, - "datatype": { - "$ref": "#/$defs/DataType" - }, - "ai_context": { - "$ref": "#/$defs/AIContext" - }, - "nonportable_extensions": { - "type": "array", - "items": { - "$ref": "#/$defs/NonportableExtension" - } - } - }, - "required": ["name", "expression"], - "additionalProperties": false - }, - "SemanticModel": { - "type": "object", - "description": "Top-level container representing a complete semantic model", - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Unique identifier for the semantic model" - }, - "description": { - "type": "string", - "description": "Human-readable description" - }, - "ai_context": { - "$ref": "#/$defs/AIContext" - }, - "datasets": { - "type": "array", - "items": { - "$ref": "#/$defs/Dataset" - }, - "minItems": 1, - "description": "Collection of logical datasets" - }, - "relationships": { - "type": "array", - "items": { - "$ref": "#/$defs/Relationship" - }, - "description": "Defines how datasets are connected" - }, - "metrics": { - "type": "array", - "items": { - "$ref": "#/$defs/Metric" - }, - "description": "Model-scoped metrics: aggregations that span datasets or need to be aware of cross-dataset concepts. Expressions reference fields by multi-part name (dataset.field) and may combine fields from more than one dataset. Names must be unique across the model. See core-spec/spec.md#metric-scoping." - }, - "nonportable_extensions": { - "type": "array", - "items": { - "$ref": "#/$defs/NonportableExtension" - } - } - }, - "required": ["name", "datasets"], - "additionalProperties": false - } - } -} diff --git a/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/paimon/spark/ossie/OssieModel.scala b/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/paimon/spark/ossie/OssieModel.scala deleted file mode 100644 index f133099ddfb3..000000000000 --- a/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/paimon/spark/ossie/OssieModel.scala +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.spark.ossie - -import org.apache.paimon.view.SemanticViewDefinition - -import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} -import com.networknt.schema.{SchemaRegistry, SpecificationVersion} -import org.yaml.snakeyaml.{LoaderOptions, Yaml} -import org.yaml.snakeyaml.constructor.SafeConstructor - -import java.nio.charset.StandardCharsets.UTF_8 -import java.util.{Collections, IdentityHashMap, Locale} - -import scala.collection.JavaConverters._ - -case class OssieField(name: String, sql: String) -case class OssieMetric(name: String, sql: String, modelScoped: Boolean) -case class OssieModel( - dataset: String, - source: String, - fields: Seq[OssieField], - metrics: Seq[OssieMetric]) - -/** Read-only execution profile of Ossie 0.3.0.dev0; the original document is never rewritten. */ -object OssieModel { - // The development version is mutable. Keep this schema and the execution profile together. - val schemaCommit: String = "2692b1cc4708fdcf6a91e597ca928a9bc9d0da60" - - private val mapper = new ObjectMapper() - - private lazy val schema = { - val stream = getClass.getResourceAsStream("ossie-schema.json") - require(stream != null, "Missing bundled Ossie schema") - try { - SchemaRegistry.withDefaultDialect(SpecificationVersion.DRAFT_2020_12).getSchema(stream) - } finally { - stream.close() - } - } - - def parse(definition: SemanticViewDefinition): OssieModel = { - require(definition.getFormat == "ossie-yaml", "Semantic query requires format ossie-yaml") - val options = new LoaderOptions() - options.setAllowDuplicateKeys(false) - val document = new Yaml(new SafeConstructor(options)).load[AnyRef](definition.getContent) - require(document != null, "Empty Ossie document") - checkJson(document) - val root = mapper.valueToTree[JsonNode](document) - val errors = schema.validate(root).asScala - require(errors.isEmpty, "Invalid Ossie 0.3.0.dev0 model: " + errors.take(5).mkString("; ")) - val datasets = elements(root, "datasets") - require(datasets.size == 1, "Ossie queries currently require exactly one dataset") - require(elements(root, "relationships").isEmpty, "Ossie relationships are not supported yet") - val dataset = datasets.head - val fields = elements(dataset, "fields").map(f => OssieField(text(f, "name"), expression(f))) - require(fields.nonEmpty, "Ossie queries require declared dataset fields") - val metrics = elements(dataset, "metrics").map( - m => OssieMetric(text(m, "name"), expression(m), modelScoped = false)) ++ - elements(root, "metrics").map( - m => OssieMetric(text(m, "name"), expression(m), modelScoped = true)) - require(metrics.nonEmpty, "Ossie queries require at least one metric") - val names = (fields.map(_.name) ++ metrics.map(_.name)).map(normalize) - require(names.distinct.size == names.size, "Ossie field and metric names must be unique") - val fieldNames = fields.map(f => normalize(f.name)).toSet - val keys = elements(dataset, "primary_key") ++ - elements(dataset, "unique_keys").flatMap(_.elements().asScala) - require( - keys.forall(k => fieldNames.contains(normalize(k.asText()))), - "Ossie keys must reference declared dataset fields") - OssieModel(text(dataset, "name"), text(dataset, "source"), fields, metrics) - } - - def normalize(name: String): String = name.toLowerCase(Locale.ROOT) - - private def expression(node: JsonNode): String = { - val expr = node.get("expression") - if (expr.has("sql")) { - text(expr, "sql") - } else { - val portable = elements(expr, "dialects").filter(d => text(d, "dialect") == "OSSIE_SQL_2026") - require(portable.size == 1, "Ossie query requires an OSSIE_SQL_2026 expression") - text(portable.head, "expression") - } - } - - private def text(node: JsonNode, name: String): String = node.get(name).asText() - - private def elements(node: JsonNode, name: String): Seq[JsonNode] = - Option(node.get(name)).map(_.elements().asScala.toVector).getOrElse(Vector.empty) - - // Preserve YAML scalar types until validation: Jackson's YAML reader turns timestamps into - // strings and map keys into strings, hiding values that Ossie forbids in extension data. - private def checkJson(root: AnyRef): Unit = { - val ancestors = Collections.newSetFromMap(new IdentityHashMap[AnyRef, java.lang.Boolean]()) - // Charge every occurrence, including aliases: a small YAML graph can expand exponentially. - // The budget bounds JSON node count plus UTF-8 scalar/key content before valueToTree allocates. - var remaining = SemanticViewDefinition.MAX_CONTENT_BYTES - def charge(size: Int): Unit = { - require(size <= remaining, "Ossie expanded YAML exceeds the supported size") - remaining -= size - } - def text(value: String): Unit = charge(value.getBytes(UTF_8).length) - def visit(value: AnyRef): Unit = { - charge(1) - value match { - case null | _: java.lang.Boolean => - case s: String => text(s) - case n: java.lang.Double => - require(java.lang.Double.isFinite(n), "Ossie values must be finite JSON numbers") - text(n.toString) - case n: java.lang.Float => - require(java.lang.Float.isFinite(n), "Ossie values must be finite JSON numbers") - text(n.toString) - case n: java.lang.Number => text(n.toString) - case map: java.util.Map[_, _] => - require(ancestors.add(map), "Ossie YAML cannot contain recursive aliases") - try { - map.asScala.foreach { - case (key, item) => - require(key.isInstanceOf[String], "Ossie JSON object keys must be strings") - text(key.asInstanceOf[String]) - visit(item.asInstanceOf[AnyRef]) - } - } finally { ancestors.remove(map) } - case list: java.util.List[_] => - require(ancestors.add(list), "Ossie YAML cannot contain recursive aliases") - try { list.asScala.foreach(item => visit(item.asInstanceOf[AnyRef])) } - finally { ancestors.remove(list) } - case _ => - throw new IllegalArgumentException("Ossie YAML values must be representable as JSON") - } - } - visit(root) - } -} diff --git a/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/catalyst/analysis/Spark42RowLevelCommandRewrite.scala b/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/catalyst/analysis/Spark42RowLevelCommandRewrite.scala deleted file mode 100644 index 1e7468e77339..000000000000 --- a/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/catalyst/analysis/Spark42RowLevelCommandRewrite.scala +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.catalyst.analysis - -import org.apache.paimon.spark.SparkTable -import org.apache.paimon.spark.catalyst.analysis.{MergeSchemaEvolutionHelper, PaimonAssignmentUtils, PaimonRelation} - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.plans.logical.{AnalysisHelper, LogicalPlan, MergeIntoTable, UpdateTable} -import org.apache.spark.sql.catalyst.rules.Rule - -/** Re-enter Spark's native rewrite with fresh, aligned commands that are not marked analyzed. */ -object Spark42RowLevelCommandRewrite - extends Rule[LogicalPlan] - with PureAppendOnlyScope - with MergeSchemaEvolutionHelper { - - override def apply(plan: LogicalPlan): LogicalPlan = { - AnalysisHelper.allowInvokingTransformsInAnalyzer { - plan.transformDown { - case u: UpdateTable if u.resolved && u.rewritable && eligible(u.table) => - val assignments = PaimonAssignmentUtils.alignUpdateAssignments( - u.table.output, - u.assignments, - fromStar = false, - mergeSchemaEnabled = false) - RewriteUpdateTable(u.copy(assignments = assignments)) - - case m: MergeIntoTable - if m.resolved && m.rewritable && !m.withSchemaEvolution && eligible(m.targetTable) => - val relation = PaimonRelation.getPaimonRelation(m.targetTable) - val table = relation.table.asInstanceOf[SparkTable] - val evolved = evolveTargetIfNeeded( - m, - relation, - table, - SparkSession.active, - _.notMatchedBySourceActions) - .map(_._1) - .getOrElse(m) - val aligned = alignAllMergeActions(evolved, evolved.targetTable.output) - RewriteMergeIntoTable(aligned.copy()) - } - } - } - - private def eligible(table: LogicalPlan): Boolean = - targetsV2CopyOnWriteTable(table) || targetsV2DeltaTable(table) -} diff --git a/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/OssieToSparkMetricView.scala b/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/OssieToSparkMetricView.scala deleted file mode 100644 index 441f785915ec..000000000000 --- a/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/OssieToSparkMetricView.scala +++ /dev/null @@ -1,305 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.paimon - -import org.apache.paimon.spark.ossie.OssieModel -import org.apache.paimon.utils.StringUtils - -import com.fasterxml.jackson.databind.ObjectMapper -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedFunction, UnresolvedStar} -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.parser.CatalystSqlParser -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.catalyst.trees.Origin -import org.apache.spark.sql.catalyst.util.DateTimeUtils -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types._ -import org.apache.spark.unsafe.types.UTF8String - -import java.util.Locale - -import scala.collection.mutable - -/** Compile field references before Spark sees the model, preserving source/field shadowing. */ -private[paimon] object OssieToSparkMetricView { - private val aggregates = Set("sum", "count", "avg", "min", "max") - private val functions = - aggregates ++ Set("abs", "round", "floor", "ceil", "ceiling", "coalesce", "date_trunc") - private val dateTruncUnits = - Set("YEAR", "QUARTER", "MONTH", "WEEK", "DAY", "HOUR", "MINUTE", "SECOND") - private val dateTruncFunction = Seq("system", "builtin", "date_trunc") - - case class Conversion(yaml: String, dateTruncOrigins: Set[Origin]) - - def convert( - spark: SparkSession, - model: OssieModel, - source: Seq[String], - sourceColumns: Seq[String]): Conversion = { - val sourceNames = sourceColumns.map(n => OssieModel.normalize(n) -> n).toMap - require(sourceNames.size == sourceColumns.size, "Ambiguous case-insensitive source columns") - val publicNames = - (model.fields.map(_.name) ++ model.metrics.map(_.name)).map(OssieModel.normalize) - val prefix = Iterator - .from(0) - .map(i => s"__ossie_${i}_") - .find(p => !publicNames.exists(_.startsWith(p))) - .get - val fieldNames = model.fields.zipWithIndex.map { - case (f, i) => OssieModel.normalize(f.name) -> s"${prefix}f$i" - }.toMap - val rawColumns = mutable.LinkedHashMap[String, String]() - - def sourceColumn(name: String): String = sourceNames.getOrElse( - OssieModel.normalize(name), - throw new IllegalArgumentException(s"Unknown Ossie source column: $name")) - - def field(name: String): Expression = UnresolvedAttribute( - Seq( - fieldNames.getOrElse( - OssieModel.normalize(name), - throw new IllegalArgumentException(s"Unknown Ossie dataset field: $name")))) - - val projections = model.fields.zipWithIndex.map { - case (f, i) => - val expr = parse(spark, f.sql, metric = false).transformUp { - case a: UnresolvedAttribute => - require(a.nameParts.size == 1, "Ossie field expressions must reference source columns") - UnresolvedAttribute(Seq(sourceColumn(a.nameParts.head))) - } - s"${render(expr)} AS ${quote(s"${prefix}f$i")}" - } - - val measures = model.metrics.map { - m => - val expr = parse(spark, m.sql, metric = true).transformUp { - case a: UnresolvedAttribute => - a.nameParts match { - case Seq(name) if !m.modelScoped => field(name) - case Seq(dataset, name) - if m.modelScoped && - OssieModel.normalize(dataset) == OssieModel.normalize(model.dataset) => - field(name) - case Seq(qualifier, name) - if !m.modelScoped && - qualifier.equalsIgnoreCase("source") && !fieldNames.contains("source") => - val actual = sourceColumn(name) - val alias = rawColumns.getOrElseUpdate(actual, s"${prefix}s${rawColumns.size}") - UnresolvedAttribute(Seq(alias)) - case _ => - throw new IllegalArgumentException(s"Invalid Ossie metric reference: ${a.sql}") - } - } - m.name -> render(expr) - } - val rawProjections = rawColumns.toSeq.map { - case (name, alias) => s"${quote(name)} AS ${quote(alias)}" - } - val sql = "SELECT " + (projections ++ rawProjections).mkString(", ") + - " FROM " + source.map(quote).mkString(".") - val mapper = new ObjectMapper() - val root = mapper.createObjectNode() - root.put("version", "0.1") - root.put("source", sql) - val dimensions = root.putArray("dimensions") - model.fields.zipWithIndex.foreach { - case (f, i) => dimensions.addObject().put("name", f.name).put("expr", quote(s"${prefix}f$i")) - } - val metricNodes = root.putArray("measures") - measures.foreach { - case (name, expr) => metricNodes.addObject().put("name", name).put("expr", expr) - } - val parser = spark.sessionState.sqlParser - val expressions = parser.parsePlan(sql).expressions ++ - measures.map { case (_, expression) => parser.parseExpression(expression) } - val dateTruncOrigins = expressions - .flatMap(_.collect { - case f: UnresolvedFunction if f.nameParts == dateTruncFunction => f.origin - }) - .toSet - // JSON is a YAML subset, and avoids another serialization path for SQL strings. - Conversion(mapper.writeValueAsString(root), dateTruncOrigins) - } - - def validateDateTruncInputs(plan: LogicalPlan, dateTruncOrigins: Set[Origin]): Unit = { - // Spark inserts the session-zone cast during analysis, including for computed arguments. - // Match model call origins so SQL UDF bodies retain their own timestamp semantics. - val truncatesLocalTimestamp = plan.exists { - node => - node.expressions.exists(_.exists { - case t @ TruncTimestamp(_, cast: Cast, _) => - dateTruncOrigins.contains(t.origin) && cast.child.dataType == TimestampNTZType - case _ => false - }) - } - require(!truncatesLocalTimestamp, "Ossie DATE_TRUNC does not support TIMESTAMP_NTZ inputs") - } - - private def parse(spark: SparkSession, sql: String, metric: Boolean): Expression = { - val conf = new SQLConf() - spark.sessionState.conf.getAllConfs.foreach { case (k, v) => conf.setConfString(k, v) } - conf.setConf(SQLConf.ANSI_ENABLED, true) - conf.setConf(SQLConf.DOUBLE_QUOTED_IDENTIFIERS, true) - conf.setConf(SQLConf.ESCAPED_STRING_LITERALS, false) - val expr = - SQLConf.withExistingConf(conf)(CatalystSqlParser.parseExpression(normalizeStrings(sql))) - var aggregate = false - expr.foreach { - case f: UnresolvedFunction => - require( - f.ignoreNulls.isEmpty && f.orderingWithinGroup.isEmpty, - "Ossie null treatment and ordered-set aggregates are not supported") - require( - f.nameParts.size == 1 && functions.contains(OssieModel.normalize(f.nameParts.head)), - s"Unsupported OSSIE_SQL_2026 function: ${f.nameParts.mkString(".")}" - ) - val name = OssieModel.normalize(f.nameParts.head) - if (name == "date_trunc") { - f.arguments match { - case Seq(Literal(unit: UTF8String, _: StringType), _) => - require( - dateTruncUnits.contains(unit.toString.toUpperCase(Locale.ROOT)), - s"Unsupported Ossie DATE_TRUNC unit: $unit") - case _ => - throw new IllegalArgumentException( - "Ossie DATE_TRUNC expects a unit string literal followed by an input expression") - } - } - if (aggregates.contains(name)) { - require(metric, "Ossie field expressions cannot contain aggregates") - aggregate = true - } - require( - !f.arguments.exists(_.isInstanceOf[UnresolvedStar]) || name == "count", - "Only COUNT(*) supports a star in Ossie expressions") - case _: Literal | _: UnresolvedAttribute | _: UnresolvedStar => - case _: Add | _: Subtract | _: Multiply | _: Divide | _: Remainder | _: UnaryMinus | - _: UnaryPositive | _: CaseWhen | _: EqualTo | _: LessThan | _: LessThanOrEqual | - _: GreaterThan | _: GreaterThanOrEqual | _: And | _: Or | _: Not | _: IsNull | - _: IsNotNull | _: In => - case c: Cast => - require( - c.dataType.isInstanceOf[NumericType] || c.dataType == StringType || - c.dataType == BooleanType || c.dataType == DateType, - s"Unsupported Ossie cast type: ${c.dataType}" - ) - case other => - throw new IllegalArgumentException(s"Unsupported OSSIE_SQL_2026 expression: ${other.sql}") - } - require(!metric || aggregate, "Ossie metrics must contain a supported aggregate") - expr - } - - // Ossie strings use ANSI doubled quotes and literal backslashes. Normalize only string tokens - // into Spark's escape syntax, leaving typed literals, identifiers and comment boundaries intact. - private def normalizeStrings(sql: String): String = { - val result = new StringBuilder - var pos = 0 - def quoted(delimiter: Char): String = { - val value = new StringBuilder - var closed = false - pos += 1 - while (pos < sql.length && !closed) { - val c = sql.charAt(pos) - pos += 1 - if (c == delimiter) { - if (pos < sql.length && sql.charAt(pos) == delimiter) { - value.append(c) - pos += 1 - } else { closed = true } - } else { value.append(c) } - } - require(closed, "Unterminated quoted Ossie expression") - value.toString - } - while (pos < sql.length) { - val c = sql.charAt(pos) - if (sql.startsWith("--", pos)) { - while (pos < sql.length && sql.charAt(pos) != '\n' && sql.charAt(pos) != '\r') { - pos += 1 - } - result.append(' ') - } else if (sql.startsWith("/*", pos)) { - var depth = 1 - pos += 2 - while (pos < sql.length && depth > 0) { - if (sql.startsWith("/*", pos)) { depth += 1; pos += 2 } - else if (sql.startsWith("*/", pos)) { depth -= 1; pos += 2 } - else { pos += 1 } - } - require(depth == 0, "Unterminated Ossie SQL comment") - result.append(' ') - } else if (c == '"') { - result.append(quote(quoted(c))) - } else if (c == '`') { - val start = pos - quoted(c) - result.append(sql.substring(start, pos)) - } else if (c == '\'') { - val rawPrefix = pos > 0 && (sql.charAt(pos - 1) == 'r' || sql.charAt(pos - 1) == 'R') && - (pos == 1 || !Character.isJavaIdentifierPart(sql.charAt(pos - 2))) - require(!rawPrefix, "Ossie requires ANSI string literals") - val value = quoted(c) - result.append('\'') - value.foreach { - case '\\' => result.append("\\\\") - case '\'' => result.append("\\'") - case other => result.append(other) - } - result.append('\'') - } else { - result.append(c) - pos += 1 - } - } - result.toString - } - - private def quote(name: String): String = "`" + name.replace("`", "``") + "`" - - // UnresolvedFunction inherits Expression.sql, which drops DISTINCT and FILTER. Render those - // explicitly before asking the surrounding scalar expressions to render their own syntax. - private def render(expr: Expression): String = expr.transformUp { - case l: Literal if l.value != null && l.dataType.isInstanceOf[StringType] => - val hex = StringUtils.byteToHexString(l.value.asInstanceOf[UTF8String].getBytes) - // Native Metric View parsing uses session settings; binary literals avoid a second round - // of string escaping regardless of escapedStringLiterals. - RenderedSQL(s"CAST(X'$hex' AS STRING)") - case Literal(micros: Long, TimestampType) => - // Use an explicit type and UTC offset so reparsing preserves the instant. - RenderedSQL(s"TIMESTAMP_LTZ '${DateTimeUtils.microsToInstant(micros)}'") - case f: UnresolvedFunction => - val nameParts = if (OssieModel.normalize(f.nameParts.head) == "date_trunc") { - dateTruncFunction - } else { - f.nameParts - } - val distinct = if (f.isDistinct) "DISTINCT " else "" - val filter = f.filter.map(p => s" FILTER (WHERE ${p.sql})").getOrElse("") - RenderedSQL( - s"${nameParts.map(quote).mkString(".")}($distinct${f.arguments.map(_.sql).mkString(", ")})$filter") - }.sql - - private case class RenderedSQL(text: String) extends LeafExpression with Unevaluable { - override def dataType: org.apache.spark.sql.types.DataType = NullType - override def nullable: Boolean = true - override def sql: String = text - } -} diff --git a/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/PaimonSemanticViewResolver.scala b/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/PaimonSemanticViewResolver.scala deleted file mode 100644 index 59cd13608f30..000000000000 --- a/paimon-spark/paimon-spark-4.3/src/main/scala/org/apache/spark/sql/paimon/PaimonSemanticViewResolver.scala +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.paimon - -import org.apache.paimon.catalog.{DelegateCatalog, Identifier => PaimonIdentifier} -import org.apache.paimon.rest.RESTCatalog -import org.apache.paimon.rest.exceptions.NoSuchResourceException -import org.apache.paimon.spark.SparkCatalog -import org.apache.paimon.spark.ossie.OssieModel -import org.apache.paimon.table.FileStoreTable -import org.apache.paimon.view.SemanticViewDefinition - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation -import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType} -import org.apache.spark.sql.catalyst.plans.logical.{CTERelationDef, CTERelationRef, Join, LateralJoin, LogicalPlan, SubqueryAlias} -import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.connector.catalog.PaimonLookupCatalog -import org.apache.spark.sql.execution.command.MetricViewHelper -import org.apache.spark.sql.metricview.util.MetricViewPlanner - -import scala.collection.JavaConverters._ -import scala.collection.mutable - -/** Resolve REST semantic definitions through Spark's Metric View engine without persisting a copy. */ -class PaimonSemanticViewResolver(spark: SparkSession) - extends Rule[LogicalPlan] - with PaimonLookupCatalog { - - protected lazy val catalogManager = spark.sessionState.catalogManager - - override def apply(plan: LogicalPlan): LogicalPlan = { - val definitions = mutable.Map[(RESTCatalog, PaimonIdentifier), Option[SemanticViewDefinition]]() - plan.resolveOperatorsDown { - case u @ UnresolvedRelation( - parts @ CatalogAndIdentifier(catalog: SparkCatalog, ident), - _, - _) => - DelegateCatalog.rootCatalog(catalog.paimonCatalog()) match { - // An ordinary view expanded by the preceding rule may introduce unresolved source - // tables after Spark's relation resolution has run. Leave those for the next iteration. - case rest: RESTCatalog if !catalog.tableExists(ident) => - val id = PaimonIdentifier.create(ident.namespace().head, ident.name()) - val definition = definitions.getOrElseUpdate( - (rest, id), - try { - Some( - rest - .semanticViewManagement() - .getSemanticView(id) - .getDefinition) - } catch { - case _: NoSuchResourceException => None - }) - definition - .map { - definition => - require(!u.isStreaming, "Ossie Semantic Views support batch queries only") - require( - u.options.get(UnresolvedRelation.REQUIRED_WRITE_PRIVILEGES) == null, - "Ossie Semantic Views are read-only") - require( - !isJoined(plan, u), - "Joins against Ossie Semantic Views are not supported yet") - val model = OssieModel.parse(definition) - val source = - spark.sessionState.sqlParser.parseMultipartIdentifier(model.source) match { - case Seq(table) => Seq(catalog.name(), ident.namespace().head, table) - case Seq(db, table) => Seq(catalog.name(), db, table) - case Seq(cat, db, table) if cat == catalog.name() => Seq(cat, db, table) - case _ => - throw new IllegalArgumentException( - "Ossie source must be a table in the same Paimon catalog") - } - val table = - catalog.paimonCatalog().getTable(PaimonIdentifier.create(source(1), source(2))) - require( - table.isInstanceOf[FileStoreTable], - "Ossie source must be a physical Paimon table") - val conversion = OssieToSparkMetricView.convert( - spark, - model, - source, - table.rowType().getFieldNames.asScala.toSeq) - val yaml = conversion.yaml - val (analyzed, _) = MetricViewHelper.analyzeMetricViewText(spark, parts, yaml) - OssieToSparkMetricView.validateDateTruncInputs( - analyzed, - conversion.dateTruncOrigins) - val metadata = CatalogTable( - identifier = TableIdentifier( - ident.name(), - Some(ident.namespace().head), - Some(catalog.name())), - tableType = CatalogTableType.VIEW, - storage = CatalogStorageFormat.empty, - schema = analyzed.schema, - viewText = Some(yaml) - ) - SubqueryAlias( - parts, - MetricViewPlanner - .planRead(metadata, yaml, spark.sessionState.sqlParser, analyzed.schema)) - } - .getOrElse(u) - case _ => u - } - } - } - - private def isJoined(plan: LogicalPlan, relation: UnresolvedRelation): Boolean = { - val ctes = plan.collectWithSubqueries { case cte: CTERelationDef => cte } - val semanticCtes = mutable.Set.empty[Long] - def dependsOnSemantic(node: LogicalPlan): Boolean = - node.collectFirstWithSubqueries { - case n if n eq relation => true - case ref: CTERelationRef if semanticCtes.contains(ref.cteId) => true - }.isDefined - - // CTE definitions are siblings of their consumers, and can reference other CTEs. Resolve - // those dependencies before inspecting joins, including joins inside scalar subqueries. - var changed = true - while (changed) { - val previousSize = semanticCtes.size - ctes.foreach(cte => if (dependsOnSemantic(cte.child)) { semanticCtes += cte.id }) - changed = semanticCtes.size != previousSize - } - plan.collectFirstWithSubqueries { - case join: Join if dependsOnSemantic(join) => true - case join: LateralJoin if dependsOnSemantic(join) => true - }.isDefined - } -} diff --git a/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/data/SparkInternalRowVariantTest.scala b/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/data/SparkInternalRowVariantTest.scala deleted file mode 100644 index fad39f3da941..000000000000 --- a/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/data/SparkInternalRowVariantTest.scala +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.spark.data - -import org.apache.paimon.data.GenericRow -import org.apache.paimon.data.variant.GenericVariant -import org.apache.paimon.types.{RowType, VariantType} - -import org.apache.spark.SparkFunSuite -import org.apache.spark.sql.types.DataTypes -import org.apache.spark.unsafe.types.VariantVal - -class SparkInternalRowVariantTest extends SparkFunSuite { - - test("get variant with generic data type access") { - val variant = GenericVariant.fromJson("""{"id":1}""") - val rowType = RowType.of(new VariantType()) - val row = SparkInternalRow.create(rowType).replace(GenericRow.of(variant)) - - val actual = row.get(0, DataTypes.VariantType).asInstanceOf[VariantVal] - - assert(actual.getValue.sameElements(variant.value())) - assert(actual.getMetadata.sameElements(variant.metadata())) - } - - test("get variant with typed access") { - val variant = GenericVariant.fromJson("""{"id":1}""") - val rowType = RowType.of(new VariantType()) - val row = SparkInternalRow.create(rowType).replace(GenericRow.of(variant)) - - // `getVariant` is implemented by `Spark4InternalRow`, while the generic `get` above goes - // through `Spark4Shim.toSparkVariant`. - val actual = row.getVariant(0) - - assert(actual.getValue.sameElements(variant.value())) - assert(actual.getMetadata.sameElements(variant.metadata())) - } -} diff --git a/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/ossie/OssieModelTest.scala b/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/ossie/OssieModelTest.scala deleted file mode 100644 index da267139fe77..000000000000 --- a/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/ossie/OssieModelTest.scala +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.spark.ossie - -import org.apache.paimon.view.SemanticViewDefinition - -import org.scalatest.funsuite.AnyFunSuite - -class OssieModelTest extends AnyFunSuite { - private val model = - """version: 0.3.0.dev0 - |name: revenue - |datasets: - | - name: orders - | source: orders - | fields: - | - name: amount - | expression: {sql: price} - | metrics: - | - name: revenue - | expression: {sql: SUM(amount)} - |""".stripMargin - - private def parse(text: String): OssieModel = - OssieModel.parse(new SemanticViewDefinition("ossie-yaml", text)) - - test("parse the flat document and ignore preserved nonportable metadata") { - val text = model + - "nonportable_extensions:\n - vendor_name: example\n data: {nested: [1, true, null]}\n" - val result = parse(text) - assert(result.dataset == "orders") - assert(result.fields == Seq(OssieField("amount", "price"))) - assert(result.metrics == Seq(OssieMetric("revenue", "SUM(amount)", modelScoped = false))) - val portable = parse( - model.replace( - "{sql: SUM(amount)}", - "{dialects: [{dialect: OSSIE_SQL_2026, expression: SUM(amount)}]}")) - assert(portable == result) - } - - test("validate the pinned schema and reject old document shapes and unknown core fields") { - Seq( - model.replace("version: 0.3.0.dev0", "version: 0.2.0.dev0"), - model + "semantic_model: []\n", - model + "custom_extensions: []\n", - model.replace("expression: {sql: price}", "expression: {sql: 42}"), - model.replace("source: orders", "source: orders\n primary_key: [missing]"), - model.replace("name: revenue\n expression", "name: AMOUNT\n expression") - ).foreach(text => intercept[IllegalArgumentException](parse(text))) - } - - test("reject multiple datasets and relationships instead of flattening joins") { - // Add the second dataset before the first dataset's fields, retaining a valid document. - val second = - model.replace("datasets:\n", "datasets:\n - name: customers\n source: customers\n") - assert(intercept[IllegalArgumentException](parse(second)).getMessage.contains("exactly one")) - val relationship = model + "relationships:\n - name: self\n from: orders\n" + - " to: orders\n from_columns: [amount]\n to_columns: [amount]\n" - assert( - intercept[IllegalArgumentException](parse(relationship)).getMessage.contains("relationships")) - } - - test("reject duplicate YAML keys, trailing documents, and nonfinite extension values") { - Seq( - model + "name: duplicate\n", - model + "---\n" + model, - model + "nonportable_extensions:\n - vendor_name: test\n data: {value: .inf}\n" - ).foreach(text => intercept[Exception](parse(text))) - } - - test("reject dialect-only expressions without a portable form") { - val text = model.replace( - "{sql: SUM(amount)}", - "{dialects: [{dialect: UNKNOWN, expression: SUM(amount)}]}") - assert(intercept[IllegalArgumentException](parse(text)).getMessage.contains("OSSIE_SQL_2026")) - assert( - intercept[IllegalArgumentException]( - OssieModel.parse(new SemanticViewDefinition("databricks-yaml", model))).getMessage - .contains("ossie-yaml")) - } - - test("extension data must be JSON values with string keys") { - Seq( - "2026-11-13", - "2026-11-13T01:02:03Z", - ".nan", - "!!binary SGVsbG8=", - "!!set {a: null}", - "&cycle [*cycle]") - .foreach { - value => - val text = - model + s"nonportable_extensions:\n - vendor_name: test\n data: {value: $value}\n" - intercept[Exception](parse(text)) - } - intercept[Exception]( - parse(model + - "nonportable_extensions:\n - vendor_name: test\n data: {123: value}\n")) - assert(parse(model + - "nonportable_extensions:\n - vendor_name: test\n data: {date: '2026-11-13', nested: [null, true, 1]}\n").dataset == "orders") - } - - test("bound expanded alias graphs and repeated scalar aliases") { - val graph = " p0: &p0 [0]\n" + (1 to 20).map { - i => s" p$i: &p$i [*p${i - 1}, *p${i - 1}]\n" - }.mkString - val scalar = " text: &text '" + ("x" * 4096) + "'\n copies: [" + - Seq.fill(300)("*text").mkString(", ") + "]\n" - Seq(graph, scalar).foreach { - data => - val text = model + "nonportable_extensions:\n - vendor_name: test\n data:\n" + data - assert(text.length < SemanticViewDefinition.MAX_CONTENT_BYTES) - val error = intercept[IllegalArgumentException](parse(text)) - assert(error.getMessage.contains("expanded YAML")) - } - val ordinaryAlias = model + - "nonportable_extensions:\n - vendor_name: test\n data: {original: &value [1, null], copy: *value}\n" - assert(parse(ordinaryAlias).dataset == "orders") - } -} diff --git a/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/sql/Spark43CompatibilityTest.scala b/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/sql/Spark43CompatibilityTest.scala index 82bd51ce35a7..94921af3e62a 100644 --- a/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/sql/Spark43CompatibilityTest.scala +++ b/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/paimon/spark/sql/Spark43CompatibilityTest.scala @@ -18,111 +18,18 @@ package org.apache.paimon.spark.sql -import org.apache.paimon.rest.RESTCatalogServer -import org.apache.paimon.rest.auth.BearTokenAuthProvider -import org.apache.paimon.rest.responses.ConfigResponse -import org.apache.paimon.spark.SparkCatalog import org.apache.paimon.spark.catalyst.parser.extensions.PaimonSpark4SqlExtensionsParser import org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions import org.apache.spark.SparkConf -import org.apache.spark.sql.{AnalysisException, QueryTest, Row} +import org.apache.spark.sql.QueryTest import org.apache.spark.sql.test.SharedSparkSession -import java.nio.file.Files -import java.util.Collections - -/** Exercise the APIs whose signatures changed in Spark 4.2 and 4.3 using real Paimon files. */ +/** Exercise the APIs that Spark 4.3 added. `Spark42CompatibilityTest` covers the rest. */ class Spark43CompatibilityTest extends QueryTest with SharedSparkSession { - private val warehouse = Files.createTempDirectory("paimon-spark43-").toFile - private lazy val server = { - val s = new RESTCatalogServer( - warehouse.toURI.toString, - new BearTokenAuthProvider("test-token"), - new ConfigResponse(Collections.singletonMap("prefix", "paimon"), Collections.emptyMap()), - "spark43-tests" - ) - s.start() - s - } override protected def sparkConf: SparkConf = super.sparkConf - .set("spark.sql.catalog.paimon", classOf[SparkCatalog].getName) - .set("spark.sql.catalog.paimon.metastore", "rest") - .set("spark.sql.catalog.paimon.uri", server.getUrl) - .set("spark.sql.catalog.paimon.warehouse", "spark43-tests") - .set("spark.sql.catalog.paimon.token.provider", "bear") - .set("spark.sql.catalog.paimon.token", "test-token") - .set("spark.sql.defaultCatalog", "paimon") .set("spark.sql.extensions", classOf[PaimonSparkSessionExtensions].getName) - // Spark 4.3 dual-runs the single-pass resolver in tests, and it cannot resolve Paimon views. - .set("spark.sql.analyzer.singlePassResolver.dualRunWithLegacy", "false") - - override protected def beforeAll(): Unit = { - super.beforeAll() - sql("CREATE DATABASE paimon.test") - sql("USE paimon.test") - } - - override protected def afterAll(): Unit = { - try { super.afterAll() } - finally { - server.shutdown() - org.apache.commons.io.FileUtils.deleteDirectory(warehouse) - } - } - - test("create, insert, CTAS and static and dynamic partition overwrite") { - withTable("t", "copied") { - sql( - "CREATE TABLE t (id INT, label VARCHAR(20), pt STRING) PARTITIONED BY (pt) " + - "TBLPROPERTIES ('bucket'='-1', 'file.format'='avro')") - sql("INSERT INTO t VALUES (1, 'a', 'x'), (2, 'b', 'y')") - sql("INSERT OVERWRITE t PARTITION (pt='x') SELECT 3, 'c'") - checkAnswer(sql("SELECT * FROM t"), Seq(Row(3, "c", "x"), Row(2, "b", "y"))) - withSQLConf("spark.sql.sources.partitionOverwriteMode" -> "dynamic") { - sql("INSERT OVERWRITE t SELECT 4, 'd', 'y'") - } - checkAnswer(sql("SELECT * FROM t"), Seq(Row(3, "c", "x"), Row(4, "d", "y"))) - sql("CREATE TABLE copied TBLPROPERTIES ('file.format'='avro') AS SELECT * FROM t") - checkAnswer(sql("SELECT * FROM copied"), sql("SELECT * FROM t")) - } - } - - test("create or replace table as select") { - withTable("t") { - sql("CREATE TABLE t (id INT, label STRING) TBLPROPERTIES ('file.format'='avro')") - sql("INSERT INTO t VALUES (1, 'old')") - sql( - "CREATE OR REPLACE TABLE t TBLPROPERTIES ('file.format'='avro') " + - "AS SELECT 2 AS id, 'new' AS label") - checkAnswer(sql("SELECT * FROM t"), Seq(Row(2, "new"))) - } - } - - test("replace table with a column list") { - withTable("t") { - sql("CREATE TABLE t (id INT, label STRING) TBLPROPERTIES ('file.format'='avro')") - sql("INSERT INTO t VALUES (1, 'old')") - sql("CREATE OR REPLACE TABLE t (id INT, score DOUBLE) TBLPROPERTIES ('file.format'='avro')") - sql("INSERT INTO t VALUES (2, 0.5)") - checkAnswer(sql("SELECT * FROM t"), Seq(Row(2, 0.5))) - } - } - - test("replace table rejects table constraints and keeps the old table") { - withTable("t") { - sql("CREATE TABLE t (id INT) TBLPROPERTIES ('file.format'='avro')") - sql("INSERT INTO t VALUES (7)") - val e = intercept[AnalysisException] { - sql( - "CREATE OR REPLACE TABLE t (id INT, CONSTRAINT positive CHECK (id > 0)) " + - "TBLPROPERTIES ('file.format'='avro')") - } - assert(e.getCondition === "UNSUPPORTED_FEATURE.TABLE_OPERATION") - checkAnswer(sql("SELECT * FROM t"), Seq(Row(7))) - } - } test("split statements through the Paimon parser") { val parser = spark.sessionState.sqlParser @@ -131,128 +38,4 @@ class Spark43CompatibilityTest extends QueryTest with SharedSparkSession { assert(result.completeStatements.map(_.statement.trim) === Seq("SELECT ';'", "SELECT 2")) assert(result.partialStatement.isEmpty) } - - test("ordinary views, table description and partition description") { - withTable("t") { - sql( - "CREATE TABLE t (id INT, pt STRING) PARTITIONED BY (pt) " + - "TBLPROPERTIES ('bucket'='-1', 'file.format'='avro')") - sql("INSERT INTO t VALUES (1, 'x')") - try { - sql("CREATE VIEW v AS SELECT id FROM t") - checkAnswer(sql("SELECT * FROM v"), Seq(Row(1))) - assert(sql("DESCRIBE t").collect().exists(_.getString(0) == "id")) - assert( - sql("DESCRIBE EXTENDED t PARTITION (pt='x')") - .collect() - .exists(_.getString(0) == "# Detailed Partition Information")) - } finally { sql("DROP VIEW IF EXISTS v") } - } - } - - test("SQL scalar functions, builtin lookup and unsupported table function rejection") { - try { - sql("CREATE FUNCTION add_one(x INT) RETURNS INT RETURN x + 1") - checkAnswer( - sql("SELECT add_one(4), SUM(abs(v)) FROM VALUES (-2), (3) AS t(v)"), - Seq(Row(5, 5))) - val unsupported = intercept[UnsupportedOperationException] { - sql("CREATE FUNCTION rows_plus(x INT) RETURNS TABLE (v INT) RETURN SELECT x + 1") - } - assert(unsupported.getMessage.contains("SQL table functions")) - } finally { - sql("DROP FUNCTION IF EXISTS add_one") - sql("DROP FUNCTION IF EXISTS rows_plus") - } - } - - test("unaliased v1 function call keeps the database-qualified column name") { - try { - sql("CREATE FUNCTION abs_udf AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFAbs'") - val df = sql("SELECT abs_udf(-3)") - checkAnswer(df, Seq(Row(3))) - assert(df.columns.toSeq === Seq("test.abs_udf(-3)")) - } finally { sql("DROP FUNCTION IF EXISTS abs_udf") } - } - - private val modes: Seq[(String, String)] = Seq( - "primary-key" -> "'bucket'='1', 'primary-key'='id'", - "append" -> "'bucket'='-1'", - "row-tracking" -> "'bucket'='-1', 'row-tracking.enabled'='true'", - "deletion-vectors" -> "'bucket'='-1', 'deletion-vectors.enabled'='true'", - "data-evolution" -> - ("'bucket'='-1', 'row-tracking.enabled'='true', 'data-evolution.enabled'='true', " + - "'deletion-vectors.enabled'='true'") - ) - - for ((mode, properties) <- modes; v2 <- Seq(false, true)) { - test(s"UPDATE, DELETE and MERGE on $mode tables with v2 writes $v2") { - withSQLConf("spark.paimon.write.use-v2-write" -> v2.toString) { - withTable("t") { - sql(s"CREATE TABLE t (id INT, v INT) TBLPROPERTIES ($properties, 'file.format'='avro')") - sql("INSERT INTO t VALUES (1, 10), (2, 20), (3, 30)") - sql("UPDATE t SET v=11 WHERE id=1") - checkAnswer(sql("SELECT * FROM t"), Seq(Row(1, 11), Row(2, 20), Row(3, 30))) - sql("DELETE FROM t WHERE id=2") - sql( - "MERGE INTO t USING (SELECT 3 AS id, 33 AS v UNION ALL SELECT 4, 44) s " + - "ON t.id=s.id WHEN MATCHED THEN UPDATE SET * " + - "WHEN NOT MATCHED THEN INSERT *") - checkAnswer(sql("SELECT * FROM t"), Seq(Row(1, 11), Row(3, 33), Row(4, 44))) - } - } - } - } - - test("native V2 rewrite handles subquery updates and insert-only merges") { - withSQLConf("spark.paimon.write.use-v2-write" -> "true") { - withTable("t") { - sql( - "CREATE TABLE t (id INT, v INT) " + - "TBLPROPERTIES ('bucket'='-1', 'file.format'='avro')") - sql("INSERT INTO t VALUES (1, 10), (2, 20)") - sql("UPDATE t SET v=99 WHERE id IN (SELECT 1)") - sql( - "MERGE INTO t USING (SELECT 3 AS id, 30 AS v) s ON t.id=s.id " + - "WHEN NOT MATCHED THEN INSERT *") - checkAnswer(sql("SELECT * FROM t"), Seq(Row(1, 99), Row(2, 20), Row(3, 30))) - sql("DELETE FROM t") - checkAnswer(sql("SELECT COUNT(*) FROM t"), Seq(Row(0L))) - } - } - } - - test("variant values and spatial scalar and array getters") { - val point = "0101000000000000000000F03F0000000000000040" - withTable("typed") { - sql( - "CREATE TABLE typed (v VARIANT, g GEOMETRY(3857), " + - "a ARRAY, gg GEOGRAPHY(4326)) " + - "TBLPROPERTIES ('bucket'='-1', 'file.format'='parquet')") - sql( - s"INSERT INTO typed SELECT parse_json('{\"id\":1}'), " + - s"ST_GeomFromWKB(unhex('$point'), 3857), " + - s"array(ST_GeomFromWKB(unhex('$point'), 3857)), ST_GeogFromWKB(unhex('$point'))") - checkAnswer( - sql( - "SELECT variant_get(v, '$.id', 'int'), hex(ST_AsBinary(g)), " + - "hex(ST_AsBinary(a[0])), hex(ST_AsBinary(gg)), ST_Srid(g), ST_Srid(a[0]) FROM typed"), - Seq(Row(1, point, point, point, 3857, 3857)) - ) - } - } - - test("spatial SQL passes a constant geometry to the Paimon scan") { - val point = - "ST_SetSrid(ST_GeomFromWKB(unhex('01010000000000000000000000000000000000F03F')), 4326)" - withTable("locations") { - sql("""CREATE TABLE locations (geom GEOMETRY(4326)) TBLPROPERTIES ( - | 'bucket' = '-1', 'row-tracking.enabled' = 'true', - | 'data-evolution.enabled' = 'true')""".stripMargin) - sql(s"INSERT INTO locations VALUES ($point), (NULL)") - val query = sql(s"SELECT ST_Srid(geom) FROM locations WHERE ST_Intersects(geom, $point)") - assert(query.queryExecution.executedPlan.toString().contains("BBoxIntersects")) - checkAnswer(query, Seq(Row(4326))) - } - } } diff --git a/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala b/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala deleted file mode 100644 index 80fd87caa11f..000000000000 --- a/paimon-spark/paimon-spark-4.3/src/test/scala/org/apache/spark/sql/paimon/OssieSemanticViewTest.scala +++ /dev/null @@ -1,746 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.paimon - -import org.apache.paimon.catalog.{DelegateCatalog, Identifier} -import org.apache.paimon.management.SemanticViewManagement -import org.apache.paimon.rest.{RESTApi, RESTCatalog, RESTCatalogServer} -import org.apache.paimon.rest.auth.{AuthProvider, BearTokenAuthProvider} -import org.apache.paimon.rest.requests.UpsertSemanticViewRequest -import org.apache.paimon.rest.responses.{ConfigResponse, GetSemanticViewResponse} -import org.apache.paimon.spark.SparkCatalog -import org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions -import org.apache.paimon.view.SemanticViewDefinition - -import okhttp3.mockwebserver.{Dispatcher, MockResponse, RecordedRequest} -import org.apache.spark.SparkConf -import org.apache.spark.sql.{QueryTest, Row} -import org.apache.spark.sql.test.SharedSparkSession -import org.apache.spark.sql.types.TimestampType - -import java.nio.file.Files -import java.time.Instant -import java.util.{Collections, Locale} -import java.util.concurrent.ConcurrentHashMap - -/** Real Spark queries, real REST client, and physical Paimon source files. */ -class OssieSemanticViewTest extends QueryTest with SharedSparkSession { - private val id = Identifier.create("sales", "order_metrics") - private val directory = Files.createTempDirectory("paimon-ossie-").toFile - private lazy val server = { - val s = new SemanticServer(directory.toURI.toString) - s.start() - s - } - private def models: SemanticViewManagement = { - val catalog = spark.sessionState.catalogManager.catalog("paimon").asInstanceOf[SparkCatalog] - DelegateCatalog - .rootCatalog(catalog.paimonCatalog()) - .asInstanceOf[RESTCatalog] - .semanticViewManagement() - } - - override protected def sparkConf: SparkConf = super.sparkConf - .set("spark.sql.catalog.paimon", classOf[SparkCatalog].getName) - .set("spark.sql.catalog.paimon.metastore", "rest") - .set("spark.sql.catalog.paimon.uri", server.getUrl) - .set("spark.sql.catalog.paimon.warehouse", "ossie-tests") - .set("spark.sql.catalog.paimon.token.provider", "bear") - .set("spark.sql.catalog.paimon.token", "test-token") - .set("spark.sql.catalog.paimon.cache-enabled", "false") - .set("spark.sql.extensions", classOf[PaimonSparkSessionExtensions].getName) - // Spark 4.3 dual-runs the single-pass resolver in tests, and it cannot resolve Paimon views. - .set("spark.sql.analyzer.singlePassResolver.dualRunWithLegacy", "false") - - override protected def beforeAll(): Unit = { - super.beforeAll() - spark.sql("CREATE DATABASE paimon.sales") - spark.sql( - "CREATE TABLE paimon.sales.orders (region_code STRING, amount DECIMAL(12, 2), " + - "discount DECIMAL(12, 2)) TBLPROPERTIES ('bucket'='-1', 'file.format'='avro')") - spark.sql( - "INSERT INTO paimon.sales.orders VALUES ('CN', 100, 10), ('CN', 100, 20), ('SG', 80, 20)") - } - - override protected def afterAll(): Unit = { - try { - super.afterAll() - } finally { - server.shutdown() - org.apache.spark.util.Utils.deleteRecursively(directory) - } - } - - private def register(yaml: String = model): Unit = { - server.semanticStatus = 200 - server.denySource = false - models.upsertSemanticView(id, new SemanticViewDefinition("ossie-yaml", yaml)) - } - - test("query dataset and model metrics with field/source shadowing and distinct aggregates") { - register() - checkAnswer( - spark.sql( - "SELECT region, MEASURE(revenue), MEASURE(gross), MEASURE(orders), " + - "MEASURE(average), MEASURE(distinct_amounts), MEASURE(model_revenue) " + - "FROM paimon.sales.order_metrics GROUP BY region"), - Seq( - Row("CN", BigDecimal(170), BigDecimal(200), 2L, BigDecimal(85), 1L, BigDecimal(170)), - Row("SG", BigDecimal(60), BigDecimal(80), 1L, BigDecimal(60), 1L, BigDecimal(60)) - ) - ) - checkAnswer( - spark.sql( - "SELECT MEASURE(revenue), MEASURE(orders) " + - "FROM paimon.sales.order_metrics WHERE region = 'CN'"), - Seq(Row(BigDecimal(170), 2L))) - checkAnswer( - spark.sql("SELECT MEASURE(revenue) FROM paimon.sales.order_metrics"), - Seq(Row(BigDecimal(230)))) - assert(server.authorizedSemanticReads > 0) - checkAnswer( - spark.sql( - "SELECT MEASURE(revenue), MEASURE(orders) " + - "FROM paimon.sales.order_metrics WHERE region = 'missing'"), - Seq(Row(null, 0L))) - } - - test("group Ossie measures by DATE_TRUNC and preserve Spark timestamp semantics") { - withSQLConf("spark.sql.session.timeZone" -> "UTC") { - spark.sql( - "CREATE TABLE paimon.sales.date_trunc_orders " + - "(order_date DATE, order_time TIMESTAMP_LTZ, amount BIGINT) " + - "TBLPROPERTIES ('bucket' = '-1')") - spark.sql("""INSERT INTO paimon.sales.date_trunc_orders VALUES - | (DATE '2026-05-10', NULL, 10), - | (DATE '2026-05-31', TIMESTAMP '2026-05-31 13:42:18.123456', 20), - | (DATE '2026-06-01', TIMESTAMP '2026-06-01 00:00:00.987654', 7), - | (NULL, NULL, 3)""".stripMargin) - - val units = Seq("year", "quarter", "month", "week", "day", "hour", "minute", "second") - val timestampFields = units - .map { - unit => - s""" - name: by_$unit - | expression: - | sql: date_trunc('${unit.toUpperCase(Locale.ROOT)}', order_time)""".stripMargin - } - .mkString("\n") - register(s"""version: 0.3.0.dev0 - |name: order_metrics - |datasets: - | - name: orders - | source: date_trunc_orders - | fields: - | - name: order_month - | expression: - | sql: DATE_TRUNC('month', order_date) - |$timestampFields - | metrics: - | - name: revenue - | expression: - | sql: SUM(source.amount) - |""".stripMargin) - - val monthly = spark.sql( - "SELECT order_month, MEASURE(revenue) AS revenue " + - "FROM paimon.sales.order_metrics GROUP BY order_month") - assert(monthly.schema("order_month").dataType == TimestampType) - checkAnswer( - monthly.selectExpr("CAST(order_month AS STRING)", "revenue"), - Seq(Row("2026-05-01 00:00:00", 30L), Row("2026-06-01 00:00:00", 7L), Row(null, 3L))) - - val columns = units.map(unit => s"by_$unit") - val grouping = columns.mkString(", ") - val timestamps = spark.sql( - s"SELECT $grouping, MEASURE(revenue) AS revenue " + - s"FROM paimon.sales.order_metrics GROUP BY $grouping") - assert(columns.forall(name => timestamps.schema(name).dataType == TimestampType)) - val formatted = columns.map(name => s"CAST($name AS STRING)") :+ "revenue" - checkAnswer( - timestamps.selectExpr(formatted: _*), - Seq( - Row( - "2026-01-01 00:00:00", - "2026-04-01 00:00:00", - "2026-05-01 00:00:00", - "2026-05-25 00:00:00", - "2026-05-31 00:00:00", - "2026-05-31 13:00:00", - "2026-05-31 13:42:00", - "2026-05-31 13:42:18", - 20L - ), - Row( - "2026-01-01 00:00:00", - "2026-04-01 00:00:00", - "2026-06-01 00:00:00", - "2026-06-01 00:00:00", - "2026-06-01 00:00:00", - "2026-06-01 00:00:00", - "2026-06-01 00:00:00", - "2026-06-01 00:00:00", - 7L - ), - Row.fromSeq(Seq.fill(columns.size)(null) :+ 13L) - ) - ) - } - } - - test("reject TIMESTAMP_NTZ inputs to Ossie DATE_TRUNC during analysis") { - spark.sql( - "CREATE TABLE paimon.sales.local_orders (order_time TIMESTAMP_NTZ, amount BIGINT) " + - "TBLPROPERTIES ('bucket' = '-1')") - spark.sql("""INSERT INTO paimon.sales.local_orders VALUES - | (TIMESTAMP_NTZ '2026-03-08 02:30:00', 10), - | (TIMESTAMP_NTZ '2026-03-08 03:30:00', 20)""".stripMargin) - val yaml = """version: 0.3.0.dev0 - |name: order_metrics - |datasets: - | - name: orders - | source: local_orders - | fields: - | - name: order_hour - | expression: - | sql: DATE_TRUNC('hour', order_time) - | metrics: - | - name: revenue - | expression: - | sql: SUM(source.amount) - |""".stripMargin - register(yaml) - val query = "SELECT order_hour, MEASURE(revenue) AS revenue " + - "FROM paimon.sales.order_metrics GROUP BY order_hour" - val message = "Ossie DATE_TRUNC does not support TIMESTAMP_NTZ inputs" - Seq("UTC", "America/Los_Angeles").foreach { - zone => - withSQLConf("spark.sql.session.timeZone" -> zone) { - val error = intercept[IllegalArgumentException] { - spark.sql(query).schema - } - assert(error.getMessage.contains(message), error.toString) - } - } - - register(yaml - .replace("DATE_TRUNC('hour', order_time)", "order_time") - .replace("SUM(source.amount)", "MAX(DATE_TRUNC('hour', COALESCE(source.order_time, NULL)))")) - val error = intercept[IllegalArgumentException] { - spark.sql("SELECT MEASURE(revenue) FROM paimon.sales.order_metrics").schema - } - assert(error.getMessage.contains(message), error.toString) - } - - test("accept Spark timestamp strings in Ossie DATE_TRUNC") { - withSQLConf("spark.sql.session.timeZone" -> "UTC") { - register(model.replace("sql: region_code", "sql: DATE_TRUNC('month', '2026-06-01')")) - val result = spark.sql( - "SELECT region, MEASURE(gross) AS gross FROM paimon.sales.order_metrics GROUP BY region") - assert(result.schema("region").dataType == TimestampType) - checkAnswer( - result.selectExpr("CAST(region AS STRING)", "gross"), - Row("2026-06-01 00:00:00", BigDecimal(280))) - } - } - - test("DATE_TRUNC input validation preserves SQL UDF body semantics") { - val session = spark.newSession() - session.conf.set("spark.sql.path.enabled", "true") - session.conf.set("spark.sql.session.timeZone", "UTC") - session.sql( - "CREATE FUNCTION spark_catalog.default.floor(ts TIMESTAMP_NTZ) " + - "RETURNS TIMESTAMP_LTZ RETURN date_trunc('day', ts)") - try { - session.sql("SET PATH = spark_catalog.default, system.builtin") - val floor = "FLOOR(TIMESTAMP_NTZ '2026-06-12 10:11:12')" - val truncate = "DATE_TRUNC('day', TIMESTAMP_LTZ '2026-06-12 10:11:12Z')" - Seq(floor, s"COALESCE($floor, $truncate)", s"DATE_TRUNC('day', $floor)").foreach { - expression => - register(model.replace("sql: region_code", s"sql: $expression")) - val result = session.sql( - "SELECT region, MEASURE(gross) AS gross FROM paimon.sales.order_metrics GROUP BY region") - assert(result.schema("region").dataType == TimestampType) - checkAnswer( - result.selectExpr("CAST(region AS STRING)", "gross"), - Row("2026-06-12 00:00:00", BigDecimal(280))) - } - register(model.replace("sql: SUM(amount)", s"sql: MAX($floor)")) - checkAnswer( - session.sql("SELECT CAST(MEASURE(revenue) AS STRING) FROM paimon.sales.order_metrics"), - Row("2026-06-12 00:00:00")) - } finally { - session.sql("DROP FUNCTION spark_catalog.default.floor") - } - } - - test("resolve Ossie DATE_TRUNC to the Spark builtin regardless of the SQL path") { - val session = spark.newSession() - session.conf.set("spark.sql.path.enabled", "true") - session.conf.set("spark.sql.session.timeZone", "UTC") - session.sql( - "CREATE FUNCTION spark_catalog.default.date_trunc(unit STRING, ts TIMESTAMP_LTZ) " + - "RETURNS STRING RETURN 'shadow'") - try { - session.sql("SET PATH = spark_catalog.default, system.builtin") - val timestamp = "TIMESTAMP_LTZ '2026-06-12 10:11:12Z'" - checkAnswer(session.sql(s"SELECT date_trunc('month', $timestamp)"), Row("shadow")) - - register(model.replace("sql: region_code", s"sql: DaTe_TrUnC('month', $timestamp)")) - val query = - "SELECT region, MEASURE(gross) AS gross FROM paimon.sales.order_metrics GROUP BY region" - val result = session.sql(query) - checkAnswer( - result.selectExpr("CAST(region AS STRING)", "gross"), - Row("2026-06-01 00:00:00", BigDecimal(280))) - assert(result.schema("region").dataType == TimestampType) - - register( - model.replace( - "sql: region_code", - "sql: DATE_TRUNC('month', TIMESTAMP_NTZ '2026-06-12 10:11:12')")) - val error = intercept[IllegalArgumentException] { - session.sql(query).schema - } - assert(error.getMessage.contains("Ossie DATE_TRUNC does not support TIMESTAMP_NTZ inputs")) - } finally { - session.sql("DROP FUNCTION spark_catalog.default.date_trunc") - } - } - - Seq("TIMESTAMP_LTZ", "TIMESTAMP_NTZ").foreach { - timestampType => - test(s"preserve Ossie DATE_TRUNC literal instants with $timestampType as the default") { - withSQLConf( - "spark.sql.timestampType" -> timestampType, - "spark.sql.session.timeZone" -> "America/Los_Angeles") { - val timestamp = "CASE WHEN region_code = 'CN' " + - "THEN TIMESTAMP_LTZ '2026-11-01 01:35:00.123456-07:00' " + - "ELSE TIMESTAMP_LTZ '2026-11-01 01:35:00.654321-08:00' END" - register(model.replace("sql: region_code", s"sql: DATE_TRUNC('hour', $timestamp)")) - val result = spark.sql( - "SELECT region, MEASURE(gross) AS gross FROM paimon.sales.order_metrics GROUP BY region") - assert(result.schema("region").dataType == TimestampType) - checkAnswer( - result.selectExpr("unix_micros(region)", "gross"), - Seq( - Row(Instant.parse("2026-11-01T08:00:00Z").getEpochSecond * 1000000L, BigDecimal(200)), - Row(Instant.parse("2026-11-01T09:00:00Z").getEpochSecond * 1000000L, BigDecimal(80)) - ) - ) - } - } - } - - test("reject invalid DATE_TRUNC units and arguments in Ossie models") { - val date = "DATE '2026-06-01'" - Seq( - s"DATE_TRUNC('mont', $date)" -> "Unsupported Ossie DATE_TRUNC unit: mont", - s"DATE_TRUNC('MM', $date)" -> "Unsupported Ossie DATE_TRUNC unit: MM", - s"DATE_TRUNC('millisecond', $date)" -> "Unsupported Ossie DATE_TRUNC unit: millisecond", - s"DATE_TRUNC('WEE\u212A', $date)" -> "Unsupported Ossie DATE_TRUNC unit: WEE\u212A", - s"DATE_TRUNC(region_code, $date)" -> "Ossie DATE_TRUNC expects a unit string literal", - s"DATE_TRUNC(NULL, $date)" -> "Ossie DATE_TRUNC expects a unit string literal", - "DATE_TRUNC('month')" -> "Ossie DATE_TRUNC expects a unit string literal" - ).foreach { - case (expression, message) => - register(model.replace("sql: region_code", s"sql: $expression")) - val error = intercept[IllegalArgumentException] { - spark.sql("SELECT region FROM paimon.sales.order_metrics").collect() - } - assert(error.getMessage.contains(message), error.toString) - } - } - - test("public field names cannot shadow private metric input aliases") { - Seq("__ossie_f1", "__ossie_0_f1").foreach { - name => - register(model.replace("name: region", s"name: $name")) - checkAnswer( - spark.sql( - s"SELECT MEASURE(revenue) FROM paimon.sales.order_metrics " + - s"WHERE $name = 'CN'"), - Seq(Row(BigDecimal(170)))) - } - } - - test("preserve FILTER, quoted identifiers, true division, and half-away-from-zero rounding") { - register( - model - .replace("sql: SUM(amount)", "sql: ROUND(SUM(amount) / COUNT(*), 0)") - .replace( - "sql: COUNT(DISTINCT source.amount)", - "sql: COUNT(DISTINCT source.amount) FILTER (WHERE region = 'CN')") - .replace( - "sql: AVG(amount)", - "sql: AVG(CASE WHEN \"region\" = 'CN' THEN \"amount\" ELSE NULL END)") - .replace("sql: SUM(source.amount)", "sql: ROUND(MIN(CAST(-2.5 AS DECIMAL(3, 1))), 0)") - .replace("sql: SUM(orders.amount)", "sql: COUNT(*) / COUNT(DISTINCT orders.region)")) - checkAnswer( - spark.sql("SELECT MEASURE(revenue), MEASURE(distinct_amounts), " + - "MEASURE(average), MEASURE(gross), MEASURE(model_revenue) FROM paimon.sales.order_metrics"), - Seq(Row(BigDecimal(77), 1L, BigDecimal(85), BigDecimal(-3), 1.5d)) - ) - } - - test("new analysis sees REST replacement and is independent of the current database") { - register() - checkAnswer( - spark.sql("SELECT MEASURE(revenue) FROM paimon.sales.order_metrics"), - Seq(Row(BigDecimal(230)))) - register(model.replace("sql: SUM(amount)", "sql: MAX(amount)")) - withSQLConf("spark.sql.caseSensitive" -> "true") { - checkAnswer( - spark.sql("SELECT MEASURE(revenue) FROM paimon.sales.order_metrics"), - Seq(Row(BigDecimal(90)))) - } - checkAnswer(spark.sql("SELECT SUM(amount) FROM paimon.sales.orders"), Seq(Row(BigDecimal(280)))) - spark.sql( - "CREATE OR REPLACE VIEW paimon.sales.ordinary AS SELECT region_code FROM paimon.sales.orders") - checkAnswer( - spark.sql("SELECT * FROM paimon.sales.ordinary"), - Seq(Row("CN"), Row("CN"), Row("SG"))) - assert(spark.sql("DESCRIBE paimon.sales.orders").collect().exists(_.getString(0) == "amount")) - spark.sql("CREATE OR REPLACE FUNCTION paimon.sales.add_one(x INT) RETURNS INT RETURN x + 1") - checkAnswer(spark.sql("SELECT paimon.sales.add_one(4)"), Seq(Row(5))) - } - - test("portable string literals keep backslashes and apostrophes under either Spark setting") { - Seq(false, true).foreach { - escaped => - withSQLConf("spark.sql.parser.escapedStringLiterals" -> escaped.toString) { - Seq("C:\\new\\tab\\%\\path's", "O'Brien", "", "路径\\new").foreach { - value => - val literal = "'" + value.replace("'", "''") + "'" - register( - model - .replace("sql: region_code", s"sql: |-\n $literal") - .replace( - "sql: COUNT(DISTINCT source.amount)", - s"sql: COUNT(*) FILTER (WHERE region = $literal)")) - checkAnswer( - spark.sql("SELECT region, MEASURE(revenue), MEASURE(distinct_amounts) " + - "FROM paimon.sales.order_metrics GROUP BY region"), - Seq(Row(value, BigDecimal(230), 3L)) - ) - } - register(model - .replace("sql: region_code", "sql: |-\n CAST(DATE '2026-11-13' AS STRING)")) - checkAnswer( - spark.sql( - "SELECT region, MEASURE(revenue) FROM paimon.sales.order_metrics GROUP BY region"), - Seq(Row("2026-11-13", BigDecimal(230)))) - register( - model.replace( - "sql: region_code", - "sql: |-\n CASE WHEN \"region_code\" = 'CN' /* ' ignored */ " + - "THEN 'O''Brien' -- ' ignored\n ELSE '路径\\new' END" - )) - checkAnswer( - spark.sql( - "SELECT region, MEASURE(revenue) FROM paimon.sales.order_metrics GROUP BY region"), - Seq(Row("O'Brien", BigDecimal(170)), Row("路径\\new", BigDecimal(60))) - ) - } - } - } - - test("CTE aliases and lateral syntax cannot hide unsupported semantic joins") { - register() - Seq( - "SELECT m.region, MEASURE(m.revenue) FROM paimon.sales.order_metrics m " + - "JOIN LATERAL (SELECT 1 AS x) t ON true GROUP BY m.region", - "WITH m AS (SELECT * FROM paimon.sales.order_metrics) " + - "SELECT MEASURE(m.revenue) FROM m JOIN paimon.sales.orders o " + - "ON m.region=o.region_code GROUP BY m.region", - "WITH m AS (SELECT * FROM paimon.sales.order_metrics), n AS (SELECT * FROM m) " + - "SELECT MEASURE(n.revenue) FROM n JOIN paimon.sales.orders o " + - "ON n.region=o.region_code GROUP BY n.region", - "WITH m AS (SELECT * FROM paimon.sales.order_metrics) " + - "SELECT (SELECT MEASURE(m.revenue) FROM m JOIN paimon.sales.orders o " + - "ON m.region=o.region_code) AS revenue" - ).foreach { - query => - val error = intercept[Exception](spark.sql(query).collect()) - assert(error.toString.contains("Joins against Ossie"), error.toString) - } - checkAnswer( - spark.sql( - "WITH m AS (SELECT MEASURE(revenue) AS revenue " + - "FROM paimon.sales.order_metrics) SELECT revenue FROM m"), - Seq(Row(BigDecimal(230)))) - checkAnswer( - spark.sql( - "WITH ordinary AS (SELECT o.region_code FROM paimon.sales.orders o " + - "JOIN paimon.sales.orders p ON o.region_code=p.region_code) " + - "SELECT MEASURE(revenue) FROM paimon.sales.order_metrics"), - Seq(Row(BigDecimal(230))) - ) - } - - test("portable SQL line comments end at CR, LF and CRLF") { - Seq("\r", "\n", "\r\n").foreach { - newline => - val expression = RESTApi.toJson("SUM(amount) -- a comment" + newline + " + COUNT(*)") - register(model.replace("sql: SUM(amount)", "sql: " + expression)) - checkAnswer( - spark.sql("SELECT MEASURE(revenue) FROM paimon.sales.order_metrics"), - Seq(Row(BigDecimal(233)))) - } - } - - test("ANSI quoted source columns retain backslashes and embedded double quotes") { - val columns = Seq("tail\\", "back\\\"quote", "apostrophe'name") - def quoted(name: String): String = "`" + name.replace("`", "``") + "`" - val schema = columns.map(name => quoted(name) + " STRING").mkString(", ") - spark.sql( - "CREATE TABLE paimon.sales.quoted_columns (" + schema + - ", amount DECIMAL(12, 2), discount DECIMAL(12, 2)) " + - "TBLPROPERTIES ('bucket'='-1', 'file.format'='parquet')") - spark.sql("INSERT INTO paimon.sales.quoted_columns VALUES ('v0', 'v1', 'v2', 100, 10)") - Seq(false, true).foreach { - escaped => - withSQLConf("spark.sql.parser.escapedStringLiterals" -> escaped.toString) { - columns.zipWithIndex.foreach { - case (column, index) => - val expression = RESTApi.toJson("\"" + column.replace("\"", "\"\"") + "\"") - register( - model - .replace("source: orders", "source: quoted_columns") - .replace("sql: region_code", "sql: " + expression)) - checkAnswer( - spark.sql( - "SELECT region, MEASURE(revenue) FROM paimon.sales.order_metrics GROUP BY region"), - Seq(Row("v" + index, BigDecimal(90)))) - } - } - } - } - - test("all semantic view write targets fail without changing the source") { - register() - val statements = Seq( - "DELETE FROM paimon.sales.order_metrics", - "UPDATE paimon.sales.order_metrics SET region='changed'", - "MERGE INTO paimon.sales.order_metrics m USING (SELECT 'CN' AS region) s " + - "ON m.region=s.region WHEN MATCHED THEN DELETE" - ) - statements.foreach { - statement => - intercept[Exception](spark.sql(statement).collect()) - checkAnswer( - spark.sql("SELECT SUM(amount), COUNT(*) FROM paimon.sales.orders"), - Seq(Row(BigDecimal(280), 3L))) - } - } - - test("preserve REST and source authorization errors") { - register() - Seq(403, 501).foreach { - status => - server.semanticStatus = status - checkAnswer( - spark.sql("SELECT SUM(amount) FROM paimon.sales.orders"), - Seq(Row(BigDecimal(280)))) - val error = intercept[Exception] { - spark.sql("SELECT MEASURE(revenue) FROM paimon.sales.order_metrics").collect() - } - assert(error.toString.contains("semantic access failed")) - } - server.semanticStatus = 200 - server.denySource = true - val error = intercept[Exception] { - spark.sql("SELECT MEASURE(revenue) FROM paimon.sales.order_metrics").collect() - } - assert(error.toString.contains("source access denied")) - server.denySource = false - val missing = intercept[Exception] { - spark.sql("SELECT * FROM paimon.sales.missing_model").collect() - } - assert(missing.toString.contains("TABLE_OR_VIEW_NOT_FOUND")) - } - - test("reject unsupported models, joins, and invalid metric references") { - Seq( - model.replace("version: 0.3.0.dev0", "version: 0.1.1") -> "Invalid Ossie", - model.replace("sql: SUM(amount)", "sql: SUM(discount)") -> "Unknown Ossie dataset field", - model.replace( - "sql: SUM(amount)", - "sql: SUM(orders.amount)") -> "Invalid Ossie metric reference", - model.replace("sql: SUM(amount)", "sql: SUM(gross)") -> "Unknown Ossie dataset field", - model.replace("sql: SUM(amount)", "sql: MEDIAN(amount)") -> "Unsupported OSSIE_SQL_2026", - model.replace("sql: SUM(amount)", "sql: SUM(amount) OVER ()") -> "Unsupported OSSIE_SQL_2026", - model.replace("sql: region_code", "sql: |-\n r'raw'") -> "ANSI string literals", - model.replace("source: orders", "source: other.sales.orders") -> "same Paimon catalog", - model.replace("source: orders", "source: order_metrics") -> "order_metrics" - ).foreach { - case (yaml, message) => - register(yaml) - val error = intercept[Exception] { - spark.sql("SELECT MEASURE(revenue) FROM paimon.sales.order_metrics").collect() - } - assert(error.toString.contains(message), error.toString) - } - register() - val join = intercept[Exception] { - spark - .sql("SELECT MEASURE(m.revenue) FROM paimon.sales.order_metrics m " + - "JOIN paimon.sales.orders o ON m.region = o.region_code GROUP BY m.region") - .collect() - } - assert(join.toString.contains("Joins against Ossie")) - val write = intercept[Exception] { - spark.sql("INSERT INTO paimon.sales.order_metrics SELECT 'CN', 1, 2, 3, 4, 5, 6, 7").collect() - } - assert( - write.toString.contains("read-only") || - write.toString.contains("TABLE_OR_VIEW_NOT_FOUND"), - write.toString) - intercept[Exception] { - spark.readStream.table("paimon.sales.order_metrics") - } - } - - test( - "source identifiers bind to the view catalog and database instead of the current namespace") { - val manager = spark.sessionState.catalogManager - val previous = manager.currentCatalog.name() +: manager.currentNamespace.toSeq - spark.sql("CREATE DATABASE paimon.elsewhere") - spark.sql( - "CREATE TABLE paimon.elsewhere.orders " + - "(region_code STRING, amount DECIMAL(12, 2), discount DECIMAL(12, 2)) " + - "TBLPROPERTIES ('bucket'='-1', 'file.format'='avro')") - spark.sql("INSERT INTO paimon.elsewhere.orders VALUES ('CN', 999, 0)") - try { - spark.sql("USE paimon.elsewhere") - Seq("orders", "sales.orders", "paimon.sales.orders").foreach { - source => - register(model.replace("source: orders", s"source: $source")) - checkAnswer( - spark.sql("SELECT MEASURE(revenue) FROM paimon.sales.order_metrics"), - Seq(Row(BigDecimal(230)))) - } - } finally { - spark.sql("USE " + previous.map(p => "`" + p.replace("`", "``") + "`").mkString(".")) - } - } - - test("analyzed queries retain their definition and queries preserve original metadata") { - val original = model + - "nonportable_extensions:\n - vendor_name: unknown\n data: {date: '2026-11-13', nested: [null, true]}\n" - register(original) - val writes = server.semanticWrites - val retained = spark.sql("SELECT MEASURE(revenue) FROM paimon.sales.order_metrics") - retained.queryExecution.analyzed - checkAnswer(retained, Seq(Row(BigDecimal(230)))) - assert(models.getSemanticView(id).getDefinition.getContent == original) - assert(server.semanticWrites == writes) - register(model.replace("sql: SUM(amount)", "sql: MAX(amount)")) - checkAnswer(retained, Seq(Row(BigDecimal(230)))) - checkAnswer( - spark.sql("SELECT MEASURE(revenue) FROM paimon.sales.order_metrics"), - Seq(Row(BigDecimal(90)))) - } - - private def model: String = - """version: 0.3.0.dev0 - |name: order_metrics - |datasets: - | - name: orders - | source: orders - | fields: - | - name: region - | expression: - | sql: region_code - | - name: amount - | expression: - | sql: amount - discount - | metrics: - | - name: revenue - | expression: - | sql: SUM(amount) - | - name: gross - | expression: - | sql: SUM(source.amount) - | - name: orders - | expression: - | sql: COUNT(*) - | - name: average - | expression: - | sql: AVG(amount) - | - name: distinct_amounts - | expression: - | sql: COUNT(DISTINCT source.amount) - |metrics: - | - name: model_revenue - | expression: - | sql: SUM(orders.amount) - |""".stripMargin -} - -private class SemanticServer(path: String) - extends RESTCatalogServer( - path, - new BearTokenAuthProvider("test-token"), - new ConfigResponse(Collections.singletonMap("prefix", "paimon"), Collections.emptyMap()), - "ossie-tests") { - private val definitions = new ConcurrentHashMap[String, SemanticViewDefinition]() - @volatile var semanticStatus: Int = 200 - @volatile var denySource: Boolean = false - @volatile var authorizedSemanticReads: Int = 0 - @volatile var semanticWrites: Int = 0 - - override def initDispatcher(auth: AuthProvider): Dispatcher = { - val delegate = super.initDispatcher(auth) - new Dispatcher { - override def dispatch(request: RecordedRequest): MockResponse = { - val route = request.getRequestUrl.encodedPath() - def response(status: Int, body: String): MockResponse = new MockResponse() - .setResponseCode(status) - .setHeader("Content-Type", "application/json") - .setBody(body) - if (route.contains("/semantic-views/")) { - val name = request.getRequestUrl - .pathSegments() - .get(request.getRequestUrl.pathSegments().size() - 1) - if (request.getMethod == "POST") { - semanticWrites += 1 - val value = - RESTApi.fromJson(request.getBody.readUtf8(), classOf[UpsertSemanticViewRequest]) - definitions.put(name, value.getDefinition) - } else { - if (semanticStatus != 200) { - return response(semanticStatus, "{\"message\":\"semantic access failed\"}") - } - if (request.getHeader("Authorization") == "Bearer test-token") { - authorizedSemanticReads += 1 - } - } - val definition = definitions.get(name) - if (definition == null) response(404, "{\"message\":\"missing semantic view\"}") - else response(200, RESTApi.toJson(new GetSemanticViewResponse(name, definition))) - } else if (denySource && route.endsWith("/tables/orders")) { - response(403, "{\"message\":\"source access denied\"}") - } else { - delegate.dispatch(request) - } - } - } - } -}