diff --git a/docs/docs/concepts/rest/rest-api.md b/docs/docs/concepts/rest/rest-api.md
index b3279b265231..c5c49b4c660b 100644
--- a/docs/docs/concepts/rest/rest-api.md
+++ b/docs/docs/concepts/rest/rest-api.md
@@ -55,6 +55,7 @@ payloads, and error responses are defined in the OpenAPI specification.
| Branches and tags | Manage named histories and retained snapshots. | Table-scoped `branches` and `tags`. |
| Consumers | List and reset streaming consumer progress. | Table-scoped `consumers`. |
| Views and functions | Manage reusable SQL and function definitions. | Database- and catalog-scoped `views` and `functions`. |
+| Prediction jobs | Submit, list, get, and cancel asynchronous predictions. | Catalog-scoped `predictions`; the source table is part of the submission. |
In this table, **table-scoped** means
`/v1/{prefix}/databases/{database}/tables/{table}`. Catalog-wide listing and detail-listing
diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml
index 40d97b3c1c7f..6791eccc76e6 100644
--- a/docs/static/rest-catalog-open-api.yaml
+++ b/docs/static/rest-catalog-open-api.yaml
@@ -133,6 +133,102 @@ paths:
"404":
$ref: '#/components/responses/DatabaseOrModelNotExistErrorResponse'
"501": {description: Models are not supported}
+ /v1/{prefix}/predictions:
+ post:
+ tags: [prediction]
+ summary: Submit a prediction job
+ operationId: submitPrediction
+ description: >-
+ Records the source snapshot and resolves the registered model. Execution reads
+ the recorded snapshot and fails if it is no longer available.
+ Reusing an idempotencyToken with the same payload returns the existing job;
+ conflicting reuse returns 409.
+ parameters:
+ - {name: prefix, in: path, required: true, schema: {type: string}}
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/SubmitPredictionRequest'
+ responses:
+ "202":
+ description: Prediction accepted
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/GetPredictionResponse'
+ "400":
+ $ref: '#/components/responses/BadRequestErrorResponse'
+ "404":
+ $ref: '#/components/responses/TableOrModelNotExistErrorResponse'
+ "409":
+ $ref: '#/components/responses/ResourceAlreadyExistErrorResponse'
+ "501": {description: Predictions are not supported}
+ get:
+ tags: [prediction]
+ summary: List prediction jobs
+ operationId: listPredictions
+ description: Newest first.
+ parameters:
+ - {name: prefix, in: path, required: true, schema: {type: string}}
+ - {name: database, in: query, required: false, schema: {type: string}}
+ - {name: table, in: query, required: false, schema: {type: string}}
+ - {name: name, in: query, required: false, schema: {type: string}}
+ - name: state
+ in: query
+ required: false
+ schema:
+ $ref: '#/components/schemas/PredictionState'
+ - {name: maxResults, in: query, required: false, schema: {type: integer, format: int32, minimum: 1}}
+ - {name: pageToken, in: query, required: false, schema: {type: string}}
+ responses:
+ "200":
+ description: Prediction jobs
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ListPredictionsResponse'
+ "501": {description: Predictions are not supported}
+ /v1/{prefix}/predictions/{predictionId}:
+ get:
+ tags: [prediction]
+ summary: Get a prediction job
+ operationId: getPrediction
+ parameters:
+ - {name: prefix, in: path, required: true, schema: {type: string}}
+ - {name: predictionId, in: path, required: true, schema: {$ref: '#/components/schemas/PredictionId'}}
+ responses:
+ "200":
+ description: Prediction metadata
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/GetPredictionResponse'
+ "404":
+ $ref: '#/components/responses/PredictionNotExistErrorResponse'
+ "501": {description: Predictions are not supported}
+ /v1/{prefix}/predictions/{predictionId}/cancel:
+ post:
+ tags: [prediction]
+ summary: Cancel a prediction job
+ operationId: cancelPrediction
+ description: >-
+ Requests cancellation and returns the latest state. Terminal jobs are
+ unchanged.
+ parameters:
+ - {name: prefix, in: path, required: true, schema: {type: string}}
+ - {name: predictionId, in: path, required: true, schema: {$ref: '#/components/schemas/PredictionId'}}
+ responses:
+ "200":
+ description: Latest prediction metadata
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/GetPredictionResponse'
+ "404":
+ $ref: '#/components/responses/PredictionNotExistErrorResponse'
+ "501": {description: Predictions are not supported}
/v1/config:
get:
tags:
@@ -2881,6 +2977,32 @@ components:
resourceType: MODEL
resourceName: model
code: 404
+ TableOrModelNotExistErrorResponse:
+ description: Database, table, or model not found
+ content:
+ application/json:
+ schema:
+ allOf:
+ - $ref: '#/components/schemas/ErrorResponse'
+ - type: object
+ required: [ resourceType ]
+ properties:
+ resourceType:
+ type: string
+ enum: [ "DATABASE", "TABLE", "MODEL" ]
+ PredictionNotExistErrorResponse:
+ description: Prediction not found
+ content:
+ application/json:
+ schema:
+ allOf:
+ - $ref: '#/components/schemas/ErrorResponse'
+ - type: object
+ required: [ resourceType ]
+ properties:
+ resourceType:
+ type: string
+ enum: [ "PREDICTION" ]
TableNotExistErrorResponse:
description:
Not Found - TableNotExistException, the table does not exist
@@ -3264,7 +3386,7 @@ components:
type: string
resourceType:
type: [ string, "null" ]
- enum: [ "DATABASE", "TABLE", "PARTITION", "COLUMN", "SNAPSHOT", "BRANCH", "TAG", "VIEW", "SEMANTIC_VIEW", "DIALECT", "FUNCTION", "MODEL", "DEFINITION", "UNKNOWN", null ]
+ enum: [ "DATABASE", "TABLE", "PARTITION", "COLUMN", "SNAPSHOT", "BRANCH", "TAG", "VIEW", "SEMANTIC_VIEW", "DIALECT", "FUNCTION", "MODEL", "PREDICTION", "DEFINITION", "UNKNOWN", null ]
resourceName:
type: [ string, "null" ]
code:
@@ -4467,6 +4589,12 @@ components:
minLength: 1
pattern: '^(?!\.{1,2}$).+'
description: A non-empty path segment other than `.` or `..`.
+ PredictionId:
+ type: string
+ minLength: 1
+ maxLength: 64
+ pattern: '^(?!\.{1,2}$).+'
+ description: A prediction identifier other than `.` or `..`.
ModelField:
type: object
additionalProperties: false
@@ -4548,6 +4676,158 @@ components:
$ref: '#/components/schemas/GetModelResponse'
nextPageToken:
type: [ string, "null" ]
+ PredictionState:
+ type: string
+ enum: [PENDING, RUNNING, TERMINATING, TERMINATED]
+ PredictionTerminationCode:
+ type: string
+ enum: [SUCCESS, SUCCESS_WITH_FAILURES, FAILED, CANCELED]
+ PredictionTerminationDetails:
+ type: object
+ additionalProperties: false
+ required: [code]
+ properties:
+ code:
+ $ref: '#/components/schemas/PredictionTerminationCode'
+ message:
+ type: string
+ PredictionStatus:
+ type: object
+ additionalProperties: false
+ required: [state]
+ properties:
+ state:
+ $ref: '#/components/schemas/PredictionState'
+ terminationDetails:
+ $ref: '#/components/schemas/PredictionTerminationDetails'
+ oneOf:
+ - properties:
+ state: {enum: [PENDING, RUNNING]}
+ not:
+ required: [terminationDetails]
+ - properties:
+ state: {const: TERMINATING}
+ - properties:
+ state: {const: TERMINATED}
+ required: [terminationDetails]
+ description: >-
+ terminationDetails is forbidden for PENDING and RUNNING, optional for
+ TERMINATING, and required for TERMINATED.
+ PredictionSource:
+ type: object
+ additionalProperties: false
+ required: [database, table]
+ properties:
+ database: {type: string, minLength: 1, maxLength: 128}
+ table: {type: string, minLength: 1, maxLength: 128}
+ PredictionOutput:
+ type: object
+ additionalProperties: false
+ required: [table]
+ properties:
+ table:
+ $ref: '#/components/schemas/PredictionSource'
+ description: >-
+ Managed result table containing source identity columns, a response string,
+ and an error string.
+ SubmitPredictionRequest:
+ type: object
+ additionalProperties: false
+ required: [source, model, inputs, instructions]
+ properties:
+ source:
+ $ref: '#/components/schemas/PredictionSource'
+ name:
+ type: string
+ minLength: 1
+ maxLength: 128
+ description: Optional non-unique label.
+ idempotencyToken:
+ type: string
+ minLength: 1
+ maxLength: 64
+ description: >-
+ Optional token reused for retries of one logical submission. SDKs generate
+ one by default; requests without a token are independent submissions.
+ model:
+ allOf:
+ - $ref: '#/components/schemas/Identifier'
+ required: [database, object]
+ properties:
+ database: {type: string, minLength: 1, maxLength: 128}
+ object: {type: string, minLength: 1, maxLength: 128}
+ inputs:
+ type: object
+ minProperties: 1
+ propertyNames: {type: string, minLength: 1, maxLength: 128}
+ additionalProperties:
+ type: string
+ minLength: 1
+ maxLength: 1024
+ description: Model input-to-column mappings.
+ mergeKeyColumns:
+ type: array
+ uniqueItems: true
+ items: {type: string, minLength: 1, maxLength: 128}
+ description: >-
+ Stable source columns used when table primary keys or row IDs are unavailable.
+ Omit to let the server derive identity; without stable identity the prediction
+ runs as a full refresh.
+ instructions:
+ type: string
+ minLength: 1
+ maxLength: 1048576
+ description: Natural-language guidance for the prediction.
+ where:
+ type: string
+ minLength: 1
+ maxLength: 1048576
+ options:
+ type: object
+ propertyNames: {type: string, minLength: 1, maxLength: 128}
+ additionalProperties:
+ type: string
+ minLength: 1
+ maxLength: 1024
+ description: >-
+ Paimon dynamic options. Use scan.snapshot-id or scan.tag-name for time
+ travel; the two options are mutually exclusive.
+ GetPredictionResponse:
+ type: object
+ additionalProperties: false
+ required: [predictionId, source, status, snapshotId, model]
+ properties:
+ predictionId: {$ref: '#/components/schemas/PredictionId'}
+ source:
+ $ref: '#/components/schemas/PredictionSource'
+ name: {type: string, minLength: 1, maxLength: 128}
+ status:
+ $ref: '#/components/schemas/PredictionStatus'
+ snapshotId: {type: integer, format: int64, minimum: 1}
+ model:
+ $ref: '#/components/schemas/Identifier'
+ output:
+ $ref: '#/components/schemas/PredictionOutput'
+ startTime:
+ type: integer
+ format: int64
+ minimum: 0
+ description: Start time as Unix epoch milliseconds in UTC.
+ endTime:
+ type: integer
+ format: int64
+ minimum: 0
+ description: End time as Unix epoch milliseconds in UTC.
+ ListPredictionsResponse:
+ type: object
+ required: [predictions]
+ properties:
+ predictions:
+ type: array
+ items:
+ $ref: '#/components/schemas/GetPredictionResponse'
+ nextPageToken:
+ type: [ string, "null" ]
ConsumerInfo:
type: object
properties:
diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java
new file mode 100644
index 000000000000..5601a02414aa
--- /dev/null
+++ b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.catalog;
+
+import org.apache.paimon.annotation.Experimental;
+
+import javax.annotation.Nullable;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/** Durable metadata for one asynchronous prediction job. */
+@Experimental
+public class Prediction {
+
+ private final String predictionId;
+ private final PredictionSource source;
+ private final @Nullable String name;
+ private final PredictionStatus status;
+ private final long snapshotId;
+ private final Identifier model;
+ private final @Nullable PredictionOutput output;
+ private final @Nullable Long startTime;
+ private final @Nullable Long endTime;
+
+ public Prediction(
+ String predictionId,
+ PredictionSource source,
+ @Nullable String name,
+ PredictionStatus status,
+ long snapshotId,
+ Identifier model,
+ @Nullable PredictionOutput output,
+ @Nullable Long startTime,
+ @Nullable Long endTime) {
+ checkNonBlank(predictionId, "predictionId");
+ checkArgument(source != null, "source must not be null");
+ checkArgument(status != null, "status must not be null");
+ checkArgument(snapshotId > 0, "snapshotId must be positive");
+ checkArgument(model != null, "model must not be null");
+ checkOptionalNonNegative(startTime, "startTime");
+ checkOptionalNonNegative(endTime, "endTime");
+ this.predictionId = predictionId;
+ this.source = source;
+ this.name = name;
+ this.status = status;
+ this.snapshotId = snapshotId;
+ this.model = model;
+ this.output = output;
+ this.startTime = startTime;
+ this.endTime = endTime;
+ }
+
+ public String predictionId() {
+ return predictionId;
+ }
+
+ public PredictionSource source() {
+ return source;
+ }
+
+ @Nullable
+ public String name() {
+ return name;
+ }
+
+ public PredictionStatus status() {
+ return status;
+ }
+
+ public long snapshotId() {
+ return snapshotId;
+ }
+
+ public Identifier model() {
+ return model;
+ }
+
+ @Nullable
+ public PredictionOutput output() {
+ return output;
+ }
+
+ /** Start time as Unix epoch milliseconds in UTC, or null if not started. */
+ @Nullable
+ public Long startTime() {
+ return startTime;
+ }
+
+ /** End time as Unix epoch milliseconds in UTC, or null if not terminated. */
+ @Nullable
+ public Long endTime() {
+ return endTime;
+ }
+
+ private static void checkNonBlank(String value, String name) {
+ checkArgument(value != null && !value.trim().isEmpty(), "%s must not be blank", name);
+ }
+
+ private static void checkOptionalNonNegative(@Nullable Long value, String name) {
+ checkArgument(value == null || value >= 0, "%s must not be negative", name);
+ }
+}
diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionOutput.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionOutput.java
new file mode 100644
index 000000000000..2ea3d165036e
--- /dev/null
+++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionOutput.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.catalog;
+
+import org.apache.paimon.annotation.Experimental;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/** Output of an asynchronous prediction. */
+@Experimental
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class PredictionOutput {
+
+ private final PredictionSource table;
+
+ @JsonCreator
+ public PredictionOutput(@JsonProperty("table") PredictionSource table) {
+ checkArgument(table != null, "table must not be null");
+ this.table = table;
+ }
+
+ @JsonGetter("table")
+ public PredictionSource table() {
+ return table;
+ }
+}
diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSource.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSource.java
new file mode 100644
index 000000000000..4b3eeeea5e50
--- /dev/null
+++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSource.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.catalog;
+
+import org.apache.paimon.annotation.Experimental;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/** Source table of a prediction job. */
+@Experimental
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class PredictionSource {
+
+ private final String database;
+ private final String table;
+
+ @JsonCreator
+ public PredictionSource(
+ @JsonProperty("database") String database, @JsonProperty("table") String table) {
+ checkNonBlank(database, "database");
+ checkNonBlank(table, "table");
+ this.database = database;
+ this.table = table;
+ }
+
+ public PredictionSource(Identifier identifier) {
+ this(identifier.getDatabaseName(), identifier.getObjectName());
+ }
+
+ @JsonGetter("database")
+ public String database() {
+ return database;
+ }
+
+ @JsonGetter("table")
+ public String table() {
+ return table;
+ }
+
+ public Identifier toIdentifier() {
+ return Identifier.create(database, table);
+ }
+
+ private static void checkNonBlank(String value, String name) {
+ checkArgument(value != null && !value.trim().isEmpty(), "%s must not be blank", name);
+ }
+}
diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java
new file mode 100644
index 000000000000..d0f98313f288
--- /dev/null
+++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.catalog;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.annotation.Experimental;
+
+import javax.annotation.Nullable;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/**
+ * Provider-neutral specification for an asynchronous prediction over a table snapshot.
+ *
+ *
{@code options} uses Paimon dynamic table option names. Time travel uses {@code
+ * scan.snapshot-id} or {@code scan.tag-name}.
+ */
+@Experimental
+public class PredictionSpec {
+
+ private final @Nullable String name;
+ private final Identifier model;
+ private final Map inputs;
+ private final List mergeKeyColumns;
+ private final String instructions;
+ private final @Nullable String where;
+ private final Map options;
+ private final @Nullable String idempotencyToken;
+
+ public PredictionSpec(
+ @Nullable String name,
+ Identifier model,
+ Map inputs,
+ @Nullable List mergeKeyColumns,
+ String instructions,
+ @Nullable String where,
+ @Nullable Map options,
+ @Nullable String idempotencyToken) {
+ checkOptionalNonBlank(name, "name");
+ checkArgument(model != null, "model must not be null");
+ checkNonBlank(model.getDatabaseName(), "model database");
+ checkNonBlank(model.getObjectName(), "model object");
+ checkNonBlank(instructions, "instructions");
+ checkOptionalNonBlank(where, "where");
+ checkOptionalNonBlank(idempotencyToken, "idempotencyToken");
+ checkArgument(
+ idempotencyToken == null || idempotencyToken.length() <= 64,
+ "idempotencyToken must not exceed 64 characters");
+ this.name = name;
+ this.model = model;
+ this.inputs = copyStringMap(inputs, "inputs", true);
+ this.mergeKeyColumns = copyOptionalStringList(mergeKeyColumns, "mergeKeyColumns");
+ this.instructions = instructions;
+ this.where = where;
+ this.idempotencyToken = idempotencyToken;
+ this.options =
+ options == null ? Collections.emptyMap() : copyStringMap(options, "options", false);
+ CoreOptions coreOptions = new CoreOptions(this.options);
+ Long snapshotId = coreOptions.scanSnapshotId();
+ String tagName = coreOptions.scanTagName();
+ checkArgument(snapshotId == null || snapshotId > 0, "scan.snapshot-id must be positive");
+ checkArgument(
+ snapshotId == null || tagName == null,
+ "scan.snapshot-id and scan.tag-name are mutually exclusive");
+ }
+
+ @Nullable
+ public String name() {
+ return name;
+ }
+
+ public Identifier model() {
+ return model;
+ }
+
+ public Map inputs() {
+ return inputs;
+ }
+
+ public List mergeKeyColumns() {
+ return mergeKeyColumns;
+ }
+
+ public String instructions() {
+ return instructions;
+ }
+
+ @Nullable
+ public String where() {
+ return where;
+ }
+
+ public Map options() {
+ return options;
+ }
+
+ @Nullable
+ public String idempotencyToken() {
+ return idempotencyToken;
+ }
+
+ private static Map copyStringMap(
+ Map values, String name, boolean nonEmpty) {
+ checkArgument(values != null, "%s must not be null", name);
+ checkArgument(!nonEmpty || !values.isEmpty(), "%s must not be empty", name);
+ Map copy = new LinkedHashMap<>();
+ values.forEach(
+ (key, value) -> {
+ checkNonBlank(key, name + " key");
+ checkNonBlank(value, name + " value");
+ copy.put(key, value);
+ });
+ return Collections.unmodifiableMap(copy);
+ }
+
+ private static List copyStringList(List values, String name) {
+ checkArgument(values != null, "%s must not be null", name);
+ List copy = new ArrayList<>();
+ for (String value : values) {
+ checkNonBlank(value, name + " value");
+ checkArgument(!copy.contains(value), "%s must not contain duplicates", name);
+ copy.add(value);
+ }
+ return Collections.unmodifiableList(copy);
+ }
+
+ private static List copyOptionalStringList(@Nullable List values, String name) {
+ return values == null ? Collections.emptyList() : copyStringList(values, name);
+ }
+
+ private static void checkOptionalNonBlank(@Nullable String value, String name) {
+ if (value != null) {
+ checkNonBlank(value, name);
+ }
+ }
+
+ private static void checkNonBlank(String value, String name) {
+ checkArgument(value != null && !value.trim().isEmpty(), "%s must not be blank", name);
+ }
+}
diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionStatus.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionStatus.java
new file mode 100644
index 000000000000..527004d191ba
--- /dev/null
+++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionStatus.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.catalog;
+
+import org.apache.paimon.annotation.Experimental;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+import javax.annotation.Nullable;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/** Lifecycle and optional termination details of an asynchronous prediction. */
+@Experimental
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class PredictionStatus {
+
+ /** Current lifecycle state. */
+ public enum State {
+ PENDING,
+ RUNNING,
+ TERMINATING,
+ TERMINATED
+ }
+
+ /** Result of a terminated prediction. */
+ public enum TerminationCode {
+ SUCCESS,
+ SUCCESS_WITH_FAILURES,
+ FAILED,
+ CANCELED
+ }
+
+ /** Result and diagnostic message available after termination. */
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ public static class TerminationDetails {
+
+ private final TerminationCode code;
+ private final @Nullable String message;
+
+ @JsonCreator
+ public TerminationDetails(
+ @JsonProperty("code") TerminationCode code,
+ @Nullable @JsonProperty("message") String message) {
+ checkArgument(code != null, "code must not be null");
+ this.code = code;
+ this.message = message;
+ }
+
+ @JsonGetter("code")
+ public TerminationCode code() {
+ return code;
+ }
+
+ @Nullable
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ @JsonGetter("message")
+ public String message() {
+ return message;
+ }
+ }
+
+ private final State state;
+ private final @Nullable TerminationDetails terminationDetails;
+
+ @JsonCreator
+ public PredictionStatus(
+ @JsonProperty("state") State state,
+ @Nullable @JsonProperty("terminationDetails") TerminationDetails terminationDetails) {
+ checkArgument(state != null, "state must not be null");
+ checkArgument(
+ state != State.TERMINATED || terminationDetails != null,
+ "terminationDetails must be present for TERMINATED state");
+ checkArgument(
+ state == State.TERMINATING
+ || state == State.TERMINATED
+ || terminationDetails == null,
+ "terminationDetails must not be present before termination starts");
+ this.state = state;
+ this.terminationDetails = terminationDetails;
+ }
+
+ @JsonGetter("state")
+ public State state() {
+ return state;
+ }
+
+ @Nullable
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ @JsonGetter("terminationDetails")
+ public TerminationDetails terminationDetails() {
+ return terminationDetails;
+ }
+
+ public boolean isTerminal() {
+ return state == State.TERMINATED;
+ }
+}
diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java
index d017d1544848..7f1ebfbd7a5a 100644
--- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java
+++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java
@@ -24,6 +24,7 @@
import org.apache.paimon.annotation.Public;
import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.catalog.PredictionStatus;
import org.apache.paimon.consumer.ConsumerInfo;
import org.apache.paimon.function.FunctionChange;
import org.apache.paimon.management.DataPolicy;
@@ -72,6 +73,7 @@
import org.apache.paimon.rest.requests.RevokePermissionRequest;
import org.apache.paimon.rest.requests.RollbackSchemaRequest;
import org.apache.paimon.rest.requests.RollbackTableRequest;
+import org.apache.paimon.rest.requests.SubmitPredictionRequest;
import org.apache.paimon.rest.requests.UpsertLabelRequest;
import org.apache.paimon.rest.requests.UpsertSemanticViewRequest;
import org.apache.paimon.rest.responses.AlterDatabaseResponse;
@@ -86,6 +88,7 @@
import org.apache.paimon.rest.responses.GetFunctionResponse;
import org.apache.paimon.rest.responses.GetLabelResponse;
import org.apache.paimon.rest.responses.GetModelResponse;
+import org.apache.paimon.rest.responses.GetPredictionResponse;
import org.apache.paimon.rest.responses.GetSchemaResponse;
import org.apache.paimon.rest.responses.GetSemanticViewResponse;
import org.apache.paimon.rest.responses.GetTableResponse;
@@ -105,6 +108,7 @@
import org.apache.paimon.rest.responses.ListPartitionsResponse;
import org.apache.paimon.rest.responses.ListPermissionsResponse;
import org.apache.paimon.rest.responses.ListPoliciesResponse;
+import org.apache.paimon.rest.responses.ListPredictionsResponse;
import org.apache.paimon.rest.responses.ListSchemasResponse;
import org.apache.paimon.rest.responses.ListSemanticViewsResponse;
import org.apache.paimon.rest.responses.ListSnapshotsResponse;
@@ -204,6 +208,8 @@ public class RESTApi {
public static final String TAG_NAME_PREFIX = "tagNamePrefix";
public static final String ENTITY_TYPE = "entityType";
public static final String ENTITY_NAME = "entityName";
+ public static final String PREDICTION_NAME = "name";
+ public static final String PREDICTION_STATE = "state";
public static final long TOKEN_EXPIRATION_SAFE_TIME_MILLIS = 3_600_000L;
@@ -1812,6 +1818,56 @@ public void dropModel(Identifier identifier) {
restAuthFunction);
}
+ /** Submit an asynchronous prediction over a table. */
+ public GetPredictionResponse submitPrediction(SubmitPredictionRequest request) {
+ return client.post(
+ resourcePaths.predictions(),
+ request,
+ GetPredictionResponse.class,
+ restAuthFunction);
+ }
+
+ /** Get one asynchronous prediction by its unique id. */
+ public GetPredictionResponse getPrediction(String predictionId) {
+ return client.get(
+ resourcePaths.prediction(predictionId),
+ GetPredictionResponse.class,
+ restAuthFunction);
+ }
+
+ /** List asynchronous predictions, ordered newest first by the server. */
+ public PagedList listPredictions(
+ @Nullable Identifier source,
+ @Nullable String name,
+ @Nullable PredictionStatus.State state,
+ @Nullable Integer maxResults,
+ @Nullable String pageToken) {
+ Map queryParams =
+ buildPagedQueryParams(
+ maxResults,
+ pageToken,
+ Pair.of("database", source == null ? null : source.getDatabaseName()),
+ Pair.of("table", source == null ? null : source.getObjectName()),
+ Pair.of(PREDICTION_NAME, name),
+ Pair.of(PREDICTION_STATE, state == null ? null : state.name()));
+ ListPredictionsResponse response =
+ client.get(
+ resourcePaths.predictions(),
+ queryParams,
+ ListPredictionsResponse.class,
+ restAuthFunction);
+ return new PagedList<>(response.getPredictions(), response.getNextPageToken());
+ }
+
+ /** Request cancellation and return the updated prediction metadata. */
+ public GetPredictionResponse cancelPrediction(String predictionId) {
+ return client.post(
+ resourcePaths.cancelPrediction(predictionId),
+ null,
+ GetPredictionResponse.class,
+ restAuthFunction);
+ }
+
/**
* Alter a function.
*
diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java
index dfbf2b489a31..8858524ca18b 100644
--- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java
+++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java
@@ -47,6 +47,7 @@ public class ResourcePaths {
protected static final String REGISTER = "register";
protected static final String FUNCTIONS = "functions";
protected static final String MODELS = "models";
+ protected static final String PREDICTIONS = "predictions";
protected static final String FUNCTION_DETAILS = "function-details";
protected static final String PERMISSIONS = "permissions";
protected static final String POLICIES = "policies";
@@ -151,16 +152,29 @@ public String database(String databaseName) {
}
public String models(String databaseName) {
- checkModelPathSegment(databaseName, "databaseName");
+ checkPathSegment(databaseName, "databaseName");
return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), MODELS);
}
public String model(String databaseName, String modelName) {
- checkModelPathSegment(modelName, "modelName");
+ checkPathSegment(modelName, "modelName");
return SLASH.join(models(databaseName), encodeString(modelName));
}
- private static void checkModelPathSegment(String value, String name) {
+ public String predictions() {
+ return SLASH.join(V1, prefix, PREDICTIONS);
+ }
+
+ public String prediction(String predictionId) {
+ checkPathSegment(predictionId, "predictionId");
+ return SLASH.join(predictions(), encodeString(predictionId));
+ }
+
+ public String cancelPrediction(String predictionId) {
+ return SLASH.join(prediction(predictionId), "cancel");
+ }
+
+ private static void checkPathSegment(String value, String name) {
checkArgument(value != null && !value.trim().isEmpty(), "%s must not be blank", name);
checkArgument(
!".".equals(value) && !"..".equals(value), "%s must not be a dot segment", name);
diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java
new file mode 100644
index 000000000000..737de6492a71
--- /dev/null
+++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.rest.requests;
+
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.catalog.PredictionSource;
+import org.apache.paimon.catalog.PredictionSpec;
+import org.apache.paimon.rest.RESTRequest;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+import javax.annotation.Nullable;
+
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
+/** Request for submitting an asynchronous prediction job. */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class SubmitPredictionRequest implements RESTRequest {
+
+ private static final String FIELD_NAME = "name";
+ private static final String FIELD_SOURCE = "source";
+ private static final String FIELD_IDEMPOTENCY_TOKEN = "idempotencyToken";
+ private static final String FIELD_MODEL = "model";
+ private static final String FIELD_INPUTS = "inputs";
+ private static final String FIELD_MERGE_KEY_COLUMNS = "mergeKeyColumns";
+ private static final String FIELD_INSTRUCTIONS = "instructions";
+ private static final String FIELD_WHERE = "where";
+ private static final String FIELD_OPTIONS = "options";
+
+ private final String idempotencyToken;
+ private final PredictionSource source;
+ private final PredictionSpec spec;
+
+ @JsonCreator
+ public SubmitPredictionRequest(
+ @JsonProperty(FIELD_SOURCE) PredictionSource source,
+ @Nullable @JsonProperty(FIELD_NAME) String name,
+ @Nullable @JsonProperty(FIELD_IDEMPOTENCY_TOKEN) String idempotencyToken,
+ @JsonProperty(FIELD_MODEL) Identifier model,
+ @JsonProperty(FIELD_INPUTS) Map inputs,
+ @Nullable @JsonProperty(FIELD_MERGE_KEY_COLUMNS) List mergeKeyColumns,
+ @JsonProperty(FIELD_INSTRUCTIONS) String instructions,
+ @Nullable @JsonProperty(FIELD_WHERE) String where,
+ @Nullable @JsonProperty(FIELD_OPTIONS) Map options) {
+ checkArgument(source != null, "source must not be null");
+ checkArgument(model != null, "model must not be null");
+ checkArgument(
+ idempotencyToken == null || !idempotencyToken.trim().isEmpty(),
+ "idempotencyToken must not be blank");
+ checkArgument(
+ idempotencyToken == null || idempotencyToken.length() <= 64,
+ "idempotencyToken must not exceed 64 characters");
+ this.source = source;
+ this.idempotencyToken =
+ idempotencyToken == null ? UUID.randomUUID().toString() : idempotencyToken;
+ this.spec =
+ new PredictionSpec(
+ name,
+ model,
+ inputs,
+ mergeKeyColumns,
+ instructions,
+ where,
+ options,
+ this.idempotencyToken);
+ }
+
+ public SubmitPredictionRequest(Identifier source, PredictionSpec spec) {
+ this(
+ new PredictionSource(source),
+ spec.name(),
+ spec.idempotencyToken(),
+ spec.model(),
+ spec.inputs(),
+ spec.mergeKeyColumns(),
+ spec.instructions(),
+ spec.where(),
+ spec.options());
+ }
+
+ @JsonGetter(FIELD_SOURCE)
+ public PredictionSource getSource() {
+ return source;
+ }
+
+ @Nullable
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ @JsonGetter(FIELD_NAME)
+ public String getName() {
+ return spec.name();
+ }
+
+ @JsonGetter(FIELD_IDEMPOTENCY_TOKEN)
+ public String getIdempotencyToken() {
+ return idempotencyToken;
+ }
+
+ @JsonGetter(FIELD_MODEL)
+ public Identifier getModel() {
+ return spec.model();
+ }
+
+ @JsonGetter(FIELD_INPUTS)
+ public Map getInputs() {
+ return spec.inputs();
+ }
+
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ @JsonGetter(FIELD_MERGE_KEY_COLUMNS)
+ public List getMergeKeyColumns() {
+ return spec.mergeKeyColumns();
+ }
+
+ @JsonGetter(FIELD_INSTRUCTIONS)
+ public String getInstructions() {
+ return spec.instructions();
+ }
+
+ @Nullable
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ @JsonGetter(FIELD_WHERE)
+ public String getWhere() {
+ return spec.where();
+ }
+
+ @JsonGetter(FIELD_OPTIONS)
+ public Map getOptions() {
+ return spec.options();
+ }
+
+ public PredictionSpec toPredictionSpec() {
+ return spec;
+ }
+}
diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java
index b5a6f1c792a7..ff47500d40f8 100644
--- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java
+++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java
@@ -57,6 +57,7 @@ public class ErrorResponse implements RESTResponse {
public static final String RESOURCE_TYPE_FUNCTION = "FUNCTION";
public static final String RESOURCE_TYPE_MODEL = "MODEL";
+ public static final String RESOURCE_TYPE_PREDICTION = "PREDICTION";
public static final String RESOURCE_TYPE_POLICY = "POLICY";
diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java
new file mode 100644
index 000000000000..c91054463443
--- /dev/null
+++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.rest.responses;
+
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.catalog.Prediction;
+import org.apache.paimon.catalog.PredictionOutput;
+import org.apache.paimon.catalog.PredictionSource;
+import org.apache.paimon.catalog.PredictionStatus;
+import org.apache.paimon.rest.RESTResponse;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+import javax.annotation.Nullable;
+
+/** Response containing durable metadata for one asynchronous prediction job. */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class GetPredictionResponse implements RESTResponse {
+
+ private final String predictionId;
+ private final PredictionSource source;
+ private final @Nullable String name;
+ private final PredictionStatus status;
+ private final long snapshotId;
+ private final Identifier model;
+ private final @Nullable PredictionOutput output;
+ private final @Nullable Long startTime;
+ private final @Nullable Long endTime;
+
+ @JsonCreator
+ public GetPredictionResponse(
+ @JsonProperty("predictionId") String predictionId,
+ @JsonProperty("source") PredictionSource source,
+ @Nullable @JsonProperty("name") String name,
+ @JsonProperty("status") PredictionStatus status,
+ @JsonProperty("snapshotId") long snapshotId,
+ @JsonProperty("model") Identifier model,
+ @Nullable @JsonProperty("output") PredictionOutput output,
+ @Nullable @JsonProperty("startTime") Long startTime,
+ @Nullable @JsonProperty("endTime") Long endTime) {
+ this.predictionId = predictionId;
+ this.source = source;
+ this.name = name;
+ this.status = status;
+ this.snapshotId = snapshotId;
+ this.model = model;
+ this.output = output;
+ this.startTime = startTime;
+ this.endTime = endTime;
+ }
+
+ public GetPredictionResponse(Prediction prediction) {
+ this(
+ prediction.predictionId(),
+ prediction.source(),
+ prediction.name(),
+ prediction.status(),
+ prediction.snapshotId(),
+ prediction.model(),
+ prediction.output(),
+ prediction.startTime(),
+ prediction.endTime());
+ }
+
+ @JsonGetter("predictionId")
+ public String getPredictionId() {
+ return predictionId;
+ }
+
+ @JsonGetter("source")
+ public PredictionSource getSource() {
+ return source;
+ }
+
+ @Nullable
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ @JsonGetter("name")
+ public String getName() {
+ return name;
+ }
+
+ @JsonGetter("status")
+ public PredictionStatus getStatus() {
+ return status;
+ }
+
+ @JsonGetter("snapshotId")
+ public long getSnapshotId() {
+ return snapshotId;
+ }
+
+ @JsonGetter("model")
+ public Identifier getModel() {
+ return model;
+ }
+
+ @Nullable
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ @JsonGetter("output")
+ public PredictionOutput getOutput() {
+ return output;
+ }
+
+ /** Start time as Unix epoch milliseconds in UTC, or null if not started. */
+ @Nullable
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ @JsonGetter("startTime")
+ public Long getStartTime() {
+ return startTime;
+ }
+
+ /** End time as Unix epoch milliseconds in UTC, or null if not terminated. */
+ @Nullable
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ @JsonGetter("endTime")
+ public Long getEndTime() {
+ return endTime;
+ }
+
+ public Prediction toPrediction() {
+ return new Prediction(
+ predictionId, source, name, status, snapshotId, model, output, startTime, endTime);
+ }
+}
diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListPredictionsResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListPredictionsResponse.java
new file mode 100644
index 000000000000..9a78780f988f
--- /dev/null
+++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListPredictionsResponse.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.rest.responses;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude;
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
+
+import javax.annotation.Nullable;
+
+import java.util.Collections;
+import java.util.List;
+
+/** Response for listing asynchronous prediction jobs. */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class ListPredictionsResponse implements PagedResponse {
+
+ private final List predictions;
+ private final @Nullable String nextPageToken;
+
+ @JsonCreator
+ public ListPredictionsResponse(
+ @Nullable @JsonProperty("predictions") List predictions,
+ @Nullable @JsonProperty("nextPageToken") String nextPageToken) {
+ this.predictions = predictions == null ? Collections.emptyList() : predictions;
+ this.nextPageToken = nextPageToken;
+ }
+
+ @JsonGetter("predictions")
+ public List getPredictions() {
+ return predictions;
+ }
+
+ @Override
+ @Nullable
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ @JsonGetter("nextPageToken")
+ public String getNextPageToken() {
+ return nextPageToken;
+ }
+
+ @Override
+ public List data() {
+ return predictions;
+ }
+}
diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiCoverageTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiCoverageTest.java
index 32da66f2abc2..e7c4d9cc9813 100644
--- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiCoverageTest.java
+++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiCoverageTest.java
@@ -79,7 +79,14 @@
class RESTApiCoverageTest {
/** RESTApi methods with no entry in DLFOpenApiActions; they are signed without an action. */
- private static final Set WITHOUT_ACTION = Collections.emptySet();
+ private static final Set WITHOUT_ACTION =
+ Collections.unmodifiableSet(
+ new TreeSet<>(
+ Arrays.asList(
+ "cancelPrediction",
+ "getPrediction",
+ "listPredictions",
+ "submitPrediction")));
/** A value per parameter type; "SELECT" is valid both as a name and as an access. */
private static final Map, Object> ARGUMENTS = arguments();
diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java
new file mode 100644
index 000000000000..9d65c53d1789
--- /dev/null
+++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java
@@ -0,0 +1,298 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.rest;
+
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.catalog.Prediction;
+import org.apache.paimon.catalog.PredictionOutput;
+import org.apache.paimon.catalog.PredictionSource;
+import org.apache.paimon.catalog.PredictionSpec;
+import org.apache.paimon.catalog.PredictionStatus;
+import org.apache.paimon.rest.requests.SubmitPredictionRequest;
+import org.apache.paimon.rest.responses.GetPredictionResponse;
+import org.apache.paimon.rest.responses.ListPredictionsResponse;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Wire-format tests for asynchronous prediction metadata. */
+class RESTApiPredictionTest {
+
+ @Test
+ void testRequestAndResponseWireFormat() throws Exception {
+ SubmitPredictionRequest request =
+ new SubmitPredictionRequest(Identifier.create("quality", "clips"), spec());
+ Map requestMap = RESTApi.fromJson(RESTApi.toJson(request), Map.class);
+ assertThat(requestMap.get("idempotencyToken")).isInstanceOf(String.class);
+ assertThat((String) requestMap.get("idempotencyToken")).isNotBlank();
+ assertThat(RESTApi.fromJson(RESTApi.toJson(request), Map.class).get("idempotencyToken"))
+ .isEqualTo(requestMap.get("idempotencyToken"));
+ assertThat(request.toPredictionSpec().idempotencyToken())
+ .isEqualTo(request.getIdempotencyToken());
+ assertThat(requestMap)
+ .containsEntry("instructions", "Classify")
+ .doesNotContainKeys(
+ "name",
+ "prompt",
+ "outputSchema",
+ "where",
+ "snapshotId",
+ "tagName",
+ "resultColumns");
+ assertThat(requestMap.get("mergeKeyColumns"))
+ .isEqualTo(Collections.singletonList("clip_id"));
+ assertThat((Map) requestMap.get("source"))
+ .containsEntry("database", "quality")
+ .containsEntry("table", "clips");
+ assertThat((Map) requestMap.get("model"))
+ .containsEntry("database", "quality")
+ .containsEntry("object", "qwen_vl");
+ assertThat((Map) requestMap.get("options"))
+ .containsEntry("scan.snapshot-id", "7");
+
+ GetPredictionResponse response = new GetPredictionResponse(prediction());
+ GetPredictionResponse decoded =
+ RESTApi.fromJson(RESTApi.toJson(response), GetPredictionResponse.class);
+ assertThat(decoded.toPrediction().status().state())
+ .isEqualTo(PredictionStatus.State.PENDING);
+ assertThat(decoded.toPrediction().status().terminationDetails()).isNull();
+ assertThat(decoded.getName()).isNull();
+ assertThat(RESTApi.fromJson(RESTApi.toJson(response), Map.class))
+ .doesNotContainKeys(
+ "createdAt",
+ "startedAt",
+ "finishedAt",
+ "modelVersion",
+ "outputTable",
+ "output",
+ "startTime",
+ "endTime");
+
+ GetPredictionResponse completed =
+ new GetPredictionResponse(
+ new Prediction(
+ "prediction-2",
+ new PredictionSource("quality", "clips"),
+ null,
+ new PredictionStatus(
+ PredictionStatus.State.TERMINATED,
+ new PredictionStatus.TerminationDetails(
+ PredictionStatus.TerminationCode.SUCCESS, null)),
+ 7,
+ Identifier.create("quality", "qwen_vl"),
+ new PredictionOutput(
+ new PredictionSource("system", "prediction_results")),
+ 10L,
+ 20L));
+ Map completedMap = RESTApi.fromJson(RESTApi.toJson(completed), Map.class);
+ assertThat(completedMap)
+ .containsEntry("startTime", 10)
+ .containsEntry("endTime", 20)
+ .doesNotContainKeys(
+ "createdAt", "startedAt", "finishedAt", "modelVersion", "outputTable");
+ Map output = (Map) completedMap.get("output");
+ assertThat((Map) output.get("table"))
+ .containsEntry("database", "system")
+ .containsEntry("table", "prediction_results");
+ assertThatThrownBy(() -> new PredictionOutput(null))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("table");
+
+ ListPredictionsResponse empty = RESTApi.fromJson("{}", ListPredictionsResponse.class);
+ assertThat(empty.getPredictions()).isEmpty();
+ assertThat(RESTApi.fromJson(RESTApi.toJson(empty), Map.class))
+ .containsOnlyKeys("predictions")
+ .containsEntry("predictions", Collections.emptyList());
+ }
+
+ @Test
+ void testStructuredModelPreservesDots() throws Exception {
+ PredictionSpec spec =
+ new PredictionSpec(
+ null,
+ Identifier.create("quality.archive", "qwen.vl"),
+ Collections.singletonMap("video", "content"),
+ null,
+ "Classify",
+ null,
+ Collections.emptyMap(),
+ "retry-1");
+ SubmitPredictionRequest request =
+ new SubmitPredictionRequest(Identifier.create("quality", "clips"), spec);
+ SubmitPredictionRequest decoded =
+ RESTApi.fromJson(RESTApi.toJson(request), SubmitPredictionRequest.class);
+
+ assertThat(decoded.getModel()).isEqualTo(Identifier.create("quality.archive", "qwen.vl"));
+ assertThat(decoded.getIdempotencyToken()).isEqualTo("retry-1");
+ }
+
+ @Test
+ void testSpecValidation() {
+ assertThatThrownBy(
+ () ->
+ new PredictionSpec(
+ null,
+ Identifier.create("quality", "qwen_vl"),
+ Collections.emptyMap(),
+ null,
+ "Classify",
+ null,
+ Collections.emptyMap(),
+ null))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("inputs must not be empty");
+
+ assertThatThrownBy(
+ () ->
+ new PredictionSpec(
+ null,
+ Identifier.create("quality", "qwen_vl"),
+ Collections.singletonMap("video", "content"),
+ Arrays.asList("clip_id", "clip_id"),
+ "Classify",
+ null,
+ Collections.emptyMap(),
+ null))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("mergeKeyColumns must not contain duplicates");
+
+ assertThatThrownBy(
+ () ->
+ new PredictionSpec(
+ null,
+ Identifier.create("quality", "qwen_vl"),
+ Collections.singletonMap("video", "content"),
+ null,
+ "Classify",
+ null,
+ Collections.singletonMap("scan.snapshot-id", "0"),
+ null))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("scan.snapshot-id must be positive");
+
+ Map conflictingOptions = new LinkedHashMap<>();
+ conflictingOptions.put("scan.snapshot-id", "7");
+ conflictingOptions.put("scan.tag-name", "training");
+ assertThatThrownBy(
+ () ->
+ new PredictionSpec(
+ null,
+ Identifier.create("quality", "qwen_vl"),
+ Collections.singletonMap("video", "content"),
+ null,
+ "Classify",
+ null,
+ conflictingOptions,
+ null))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("mutually exclusive");
+
+ assertThatThrownBy(
+ () ->
+ new PredictionSpec(
+ null,
+ Identifier.create("", "qwen_vl"),
+ Collections.singletonMap("video", "content"),
+ null,
+ "Classify",
+ null,
+ Collections.emptyMap(),
+ null))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("model database");
+
+ assertThatThrownBy(
+ () ->
+ new PredictionSpec(
+ null,
+ Identifier.create("quality", "qwen_vl"),
+ Collections.singletonMap("video", "content"),
+ null,
+ "Classify",
+ null,
+ Collections.emptyMap(),
+ String.join("", Collections.nCopies(65, "x"))))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("64 characters");
+ }
+
+ @Test
+ void testTerminalStates() {
+ assertThat(new PredictionStatus(PredictionStatus.State.PENDING, null).isTerminal())
+ .isFalse();
+ assertThat(new PredictionStatus(PredictionStatus.State.RUNNING, null).isTerminal())
+ .isFalse();
+ assertThat(new PredictionStatus(PredictionStatus.State.TERMINATING, null).isTerminal())
+ .isFalse();
+ for (PredictionStatus.TerminationCode code :
+ Arrays.asList(
+ PredictionStatus.TerminationCode.CANCELED,
+ PredictionStatus.TerminationCode.SUCCESS)) {
+ PredictionStatus terminated =
+ new PredictionStatus(
+ PredictionStatus.State.TERMINATED,
+ new PredictionStatus.TerminationDetails(code, null));
+ assertThat(terminated.isTerminal()).isTrue();
+ assertThat(terminated.terminationDetails().code()).isEqualTo(code);
+ }
+ assertThatThrownBy(() -> new PredictionStatus(PredictionStatus.State.TERMINATED, null))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThatThrownBy(
+ () ->
+ new PredictionStatus(
+ PredictionStatus.State.RUNNING,
+ new PredictionStatus.TerminationDetails(
+ PredictionStatus.TerminationCode.SUCCESS, null)))
+ .isInstanceOf(IllegalArgumentException.class);
+ }
+
+ private static PredictionSpec spec() {
+ Map inputs = new LinkedHashMap<>();
+ inputs.put("video", "content");
+ return new PredictionSpec(
+ null,
+ Identifier.create("quality", "qwen_vl"),
+ inputs,
+ Collections.singletonList("clip_id"),
+ "Classify",
+ null,
+ Collections.singletonMap("scan.snapshot-id", "7"),
+ null);
+ }
+
+ private static Prediction prediction() {
+ return new Prediction(
+ "prediction-1",
+ new PredictionSource("quality", "clips"),
+ null,
+ new PredictionStatus(PredictionStatus.State.PENDING, null),
+ 7,
+ Identifier.create("quality", "qwen_vl"),
+ null,
+ null,
+ null);
+ }
+}
diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiActionsTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiActionsTest.java
index 9e4a4f1c36b3..d456239417fe 100644
--- a/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiActionsTest.java
+++ b/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiActionsTest.java
@@ -26,6 +26,7 @@
import org.junit.jupiter.api.Test;
import java.io.IOException;
+import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
import java.util.TreeSet;
@@ -38,6 +39,15 @@
/** Test for {@link DLFOpenApiActions}. */
public class DLFOpenApiActionsTest {
+ private static final Set REST_ONLY_OPERATIONS =
+ new HashSet<>(
+ Arrays.asList(
+ OpenApiSpec.normalize("POST /v1/{prefix}/predictions"),
+ OpenApiSpec.normalize("GET /v1/{prefix}/predictions"),
+ OpenApiSpec.normalize("GET /v1/{prefix}/predictions/{predictionId}"),
+ OpenApiSpec.normalize(
+ "POST /v1/{prefix}/predictions/{predictionId}/cancel")));
+
private final ResourcePaths paths = new ResourcePaths("clg-paimon-1");
/** Each template, filled in, must come back to its own action rather than an earlier one. */
@@ -59,12 +69,13 @@ public void testOperationsMatchTheOpenApiSpec() throws IOException {
for (String operation : OpenApiSpec.operations()) {
spec.add(OpenApiSpec.normalize(operation));
}
+ spec.removeAll(REST_ONLY_OPERATIONS);
Set registered = new TreeSet<>();
for (String[] operation : DLFOpenApiActions.operations()) {
registered.add(OpenApiSpec.normalize(operation[0] + " " + operation[1]));
}
assertThat(registered)
- .as("DLFOpenApiActions must register exactly the spec operations")
+ .as("DLFOpenApiActions must register the POP-enabled spec operations")
.containsExactlyInAnyOrderElementsOf(spec);
}
diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java
index 3c2c64357630..248c6d75c115 100644
--- a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java
+++ b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java
@@ -225,7 +225,8 @@ public class RequestJacksonCompatibilityTest {
RenameTableRequest.class,
ReplaceTableRequest.class,
RevokePermissionRequest.class,
- RollbackTableRequest.class)
+ RollbackTableRequest.class,
+ SubmitPredictionRequest.class)
.collect(Collectors.toSet());
@ParameterizedTest(name = "{0}")
diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java
index 0aa2795bad9e..0eae9bb8f99a 100644
--- a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java
@@ -1197,6 +1197,34 @@ default void dropModel(Identifier identifier, boolean ignoreIfNotExists)
throw new UnsupportedOperationException("Current Catalog does not support models");
}
+ // ======================= Prediction methods ===============================
+
+ /** Submit an asynchronous prediction over a table and its fixed snapshot. */
+ default Prediction submitPrediction(Identifier identifier, PredictionSpec spec)
+ throws DatabaseNotExistException, TableNotExistException, ModelNotExistException {
+ throw new UnsupportedOperationException("Current Catalog does not support predictions");
+ }
+
+ /** Get one asynchronous prediction by its unique id. */
+ default Prediction getPrediction(String predictionId) throws PredictionNotExistException {
+ throw new UnsupportedOperationException("Current Catalog does not support predictions");
+ }
+
+ /** List asynchronous predictions, optionally filtered by source table. */
+ default PagedList listPredictions(
+ @Nullable Identifier source,
+ @Nullable String name,
+ @Nullable PredictionStatus.State state,
+ @Nullable Integer maxResults,
+ @Nullable String pageToken) {
+ throw new UnsupportedOperationException("Current Catalog does not support predictions");
+ }
+
+ /** Request cancellation and return the updated prediction metadata. */
+ default Prediction cancelPrediction(String predictionId) throws PredictionNotExistException {
+ throw new UnsupportedOperationException("Current Catalog does not support predictions");
+ }
+
// ======================= Function methods ===============================
/**
@@ -1937,6 +1965,27 @@ public Identifier identifier() {
}
}
+ /** Exception for trying to access a prediction that doesn't exist. */
+ class PredictionNotExistException extends Exception {
+
+ private static final String MSG = "Prediction %s doesn't exist.";
+
+ private final String predictionId;
+
+ public PredictionNotExistException(String predictionId) {
+ this(predictionId, null);
+ }
+
+ public PredictionNotExistException(String predictionId, @Nullable Throwable cause) {
+ super(String.format(MSG, predictionId), cause);
+ this.predictionId = predictionId;
+ }
+
+ public String predictionId() {
+ return predictionId;
+ }
+ }
+
/** Exception for trying to create a function that already exists. */
class FunctionAlreadyExistException extends Exception {
diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java
index 48383b090066..8b72c9496767 100644
--- a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java
@@ -407,6 +407,32 @@ public void dropModel(Identifier identifier, boolean ignoreIfNotExists)
wrapped.dropModel(identifier, ignoreIfNotExists);
}
+ @Override
+ public Prediction submitPrediction(Identifier identifier, PredictionSpec spec)
+ throws DatabaseNotExistException, TableNotExistException, ModelNotExistException {
+ return wrapped.submitPrediction(identifier, spec);
+ }
+
+ @Override
+ public Prediction getPrediction(String predictionId) throws PredictionNotExistException {
+ return wrapped.getPrediction(predictionId);
+ }
+
+ @Override
+ public PagedList listPredictions(
+ @Nullable Identifier source,
+ @Nullable String name,
+ @Nullable PredictionStatus.State state,
+ @Nullable Integer maxResults,
+ @Nullable String pageToken) {
+ return wrapped.listPredictions(source, name, state, maxResults, pageToken);
+ }
+
+ @Override
+ public Prediction cancelPrediction(String predictionId) throws PredictionNotExistException {
+ return wrapped.cancelPrediction(predictionId);
+ }
+
@Override
public List listFunctions(String databaseName) throws DatabaseNotExistException {
return wrapped.listFunctions(databaseName);
diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java
index fd5a0df13732..d06405196110 100644
--- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java
+++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java
@@ -31,6 +31,9 @@
import org.apache.paimon.catalog.Database;
import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.catalog.Model;
+import org.apache.paimon.catalog.Prediction;
+import org.apache.paimon.catalog.PredictionSpec;
+import org.apache.paimon.catalog.PredictionStatus;
import org.apache.paimon.catalog.PropertyChange;
import org.apache.paimon.catalog.TableMetadata;
import org.apache.paimon.catalog.TableQueryAuthResult;
@@ -58,6 +61,7 @@
import org.apache.paimon.rest.exceptions.NotImplementedException;
import org.apache.paimon.rest.exceptions.ServiceFailureException;
import org.apache.paimon.rest.requests.CreateModelRequest;
+import org.apache.paimon.rest.requests.SubmitPredictionRequest;
import org.apache.paimon.rest.responses.AuthTableQueryResponse;
import org.apache.paimon.rest.responses.ErrorResponse;
import org.apache.paimon.rest.responses.GetDatabaseResponse;
@@ -1210,6 +1214,84 @@ public void dropModel(Identifier identifier, boolean ignoreIfNotExists)
}
}
+ @Override
+ public Prediction submitPrediction(Identifier identifier, PredictionSpec spec)
+ throws DatabaseNotExistException, TableNotExistException, ModelNotExistException {
+ checkNotBranch(identifier, "submitPrediction");
+ checkNotSystemTable(identifier, "submitPrediction");
+ try {
+ return api.submitPrediction(new SubmitPredictionRequest(identifier, spec))
+ .toPrediction();
+ } catch (NoSuchResourceException e) {
+ if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_DATABASE)) {
+ String databaseName =
+ StringUtils.isNullOrWhitespaceOnly(e.resourceName())
+ ? identifier.getDatabaseName()
+ : e.resourceName();
+ throw new DatabaseNotExistException(databaseName, e);
+ }
+ if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_MODEL)) {
+ throw new ModelNotExistException(spec.model(), e);
+ }
+ if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_TABLE)) {
+ throw new TableNotExistException(identifier, e);
+ }
+ throw e;
+ } catch (NotImplementedException e) {
+ throw new UnsupportedOperationException("REST Catalog does not support predictions", e);
+ }
+ }
+
+ @Override
+ public Prediction getPrediction(String predictionId) throws PredictionNotExistException {
+ try {
+ return api.getPrediction(predictionId).toPrediction();
+ } catch (NoSuchResourceException e) {
+ if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_PREDICTION)) {
+ throw new PredictionNotExistException(predictionId, e);
+ }
+ throw e;
+ } catch (NotImplementedException e) {
+ throw new UnsupportedOperationException("REST Catalog does not support predictions", e);
+ }
+ }
+
+ @Override
+ public PagedList listPredictions(
+ @Nullable Identifier source,
+ @Nullable String name,
+ @Nullable PredictionStatus.State state,
+ @Nullable Integer maxResults,
+ @Nullable String pageToken) {
+ try {
+ PagedList response =
+ api.listPredictions(source, name, state, maxResults, pageToken);
+ return new PagedList<>(
+ response.getElements().stream()
+ .map(
+ org.apache.paimon.rest.responses.GetPredictionResponse
+ ::toPrediction)
+ .collect(Collectors.toList()),
+ response.getNextPageToken());
+ } catch (NotImplementedException e) {
+ throw new UnsupportedOperationException("REST Catalog does not support predictions", e);
+ }
+ }
+
+ @Override
+ public Prediction cancelPrediction(String predictionId) throws PredictionNotExistException {
+ try {
+ return api.cancelPrediction(predictionId).toPrediction();
+ } catch (NoSuchResourceException e) {
+ if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_PREDICTION)) {
+ throw new PredictionNotExistException(predictionId, e);
+ }
+ throw e;
+ } catch (NotImplementedException e) {
+ throw new UnsupportedOperationException("REST Catalog does not support predictions", e);
+ }
+ }
+
@Override
public List listFunctions(String databaseName) throws DatabaseNotExistException {
try {
diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java
index ecf653f58c0f..bdec50a83960 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java
@@ -27,8 +27,13 @@
import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.catalog.Model;
import org.apache.paimon.catalog.ModelField;
+import org.apache.paimon.catalog.Prediction;
+import org.apache.paimon.catalog.PredictionSpec;
+import org.apache.paimon.catalog.PredictionStatus;
import org.apache.paimon.catalog.ProviderModelDefinition;
import org.apache.paimon.catalog.TableQueryAuthResult;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.GenericRow;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.management.ListPermissionsRequest;
@@ -72,6 +77,7 @@
import org.apache.paimon.table.sink.BatchTableCommit;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.types.RowKind;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.BuildVersions;
import org.apache.paimon.utils.InstantiationUtil;
@@ -91,6 +97,7 @@
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Paths;
+import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -120,6 +127,7 @@
class MockRESTCatalogTest extends RESTCatalogTest {
private RESTCatalogServer restCatalogServer;
+ private RESTCatalogPredictionTestControl predictionTestControl;
private final String serverDefineHeaderName = "test-header";
private final String serverDefineHeaderValue = "test-value";
private String dataPath;
@@ -190,6 +198,382 @@ void testModelCrudRoundTrip() throws Exception {
.isInstanceOf(Catalog.DatabaseNotExistException.class);
}
+ @Test
+ void testPredictionReferenceIntegration() throws Exception {
+ String database = "prediction_reference";
+ Identifier sourceIdentifier = Identifier.create(database, "clips");
+ Identifier modelIdentifier = Identifier.create(database, "echo_model");
+ catalog.createDatabase(database, false);
+ catalog.createTable(
+ sourceIdentifier,
+ Schema.newBuilder()
+ .column("id", DataTypes.INT())
+ .column("content", DataTypes.STRING())
+ .build(),
+ false);
+ FileStoreTable source = (FileStoreTable) catalog.getTable(sourceIdentifier);
+ commitRows(source, GenericRow.of(1, BinaryString.fromString("first")));
+
+ ProviderModelDefinition definition =
+ new ProviderModelDefinition(
+ "reference",
+ "echo",
+ Collections.singletonList(new ModelField("text", "STRING", null)),
+ Collections.singletonList(new ModelField("response", "STRING", null)),
+ Collections.emptyMap());
+ catalog.createModel(
+ modelIdentifier,
+ new Model(modelIdentifier, "Deterministic reference model", definition),
+ false);
+
+ Runnable resumePredictionExecution = predictionTestControl.pauseExecution();
+ Prediction submitted =
+ catalog.submitPrediction(
+ sourceIdentifier,
+ new PredictionSpec(
+ "snapshot-readback",
+ modelIdentifier,
+ Collections.singletonMap("text", "content"),
+ Collections.singletonList("id"),
+ "Echo the input",
+ null,
+ Collections.emptyMap(),
+ "reference-success"));
+ long pinnedSnapshot = submitted.snapshotId();
+
+ commitRows(source, GenericRow.of(2, BinaryString.fromString("second")));
+ assertThat(
+ ((FileStoreTable) catalog.getTable(sourceIdentifier))
+ .snapshotManager()
+ .latestSnapshotId())
+ .isGreaterThan(pinnedSnapshot);
+
+ resumePredictionExecution.run();
+ Prediction completed = waitForPrediction(submitted.predictionId());
+ assertThat(completed.status().terminationDetails().code())
+ .isEqualTo(PredictionStatus.TerminationCode.SUCCESS);
+ assertThat(completed.snapshotId()).isEqualTo(pinnedSnapshot);
+ assertThat(completed.output()).isNotNull();
+ assertThat(batchRead(catalog.getTable(completed.output().table().toIdentifier())))
+ .containsExactly("+I[1, {\"text\":\"first\"}, null]");
+
+ resumePredictionExecution = predictionTestControl.pauseExecution();
+ Prediction toCancel =
+ catalog.submitPrediction(
+ sourceIdentifier,
+ new PredictionSpec(
+ "cancel-reference",
+ modelIdentifier,
+ Collections.singletonMap("text", "content"),
+ Collections.singletonList("id"),
+ "Echo the input",
+ null,
+ Collections.emptyMap(),
+ "reference-cancel"));
+ waitForPredictionState(toCancel.predictionId(), PredictionStatus.State.RUNNING);
+ Prediction canceling = catalog.cancelPrediction(toCancel.predictionId());
+ assertThat(canceling.status().state()).isEqualTo(PredictionStatus.State.TERMINATING);
+ resumePredictionExecution.run();
+ Prediction canceled = waitForPrediction(toCancel.predictionId());
+ assertThat(canceled.status().terminationDetails().code())
+ .isEqualTo(PredictionStatus.TerminationCode.CANCELED);
+ assertThat(canceled.output()).isNull();
+ }
+
+ @Test
+ void testConcurrentIdempotentPredictionSubmissionsReturnSameJob() throws Exception {
+ PredictionFixture fixture = createPredictionFixture("prediction_concurrent_same");
+ PredictionSpec spec = fixture.spec("same-request", "same-token");
+ Runnable resumePredictionExecution = predictionTestControl.pauseExecution();
+ predictionTestControl.synchronizeNextSubmissions(2);
+ ExecutorService submissions = Executors.newFixedThreadPool(2);
+ try {
+ Future first =
+ submissions.submit(() -> catalog.submitPrediction(fixture.source, spec));
+ Future second =
+ submissions.submit(() -> catalog.submitPrediction(fixture.source, spec));
+
+ assertThat(first.get(10, TimeUnit.SECONDS).predictionId())
+ .isEqualTo(second.get(10, TimeUnit.SECONDS).predictionId());
+ } finally {
+ resumePredictionExecution.run();
+ submissions.shutdownNow();
+ }
+ }
+
+ @Test
+ void testConcurrentConflictingPredictionSubmissionsReturnConflict() throws Exception {
+ PredictionFixture fixture = createPredictionFixture("prediction_concurrent_conflict");
+ PredictionSpec firstSpec = fixture.spec("first-request", "conflict-token");
+ PredictionSpec secondSpec = fixture.spec("second-request", "conflict-token");
+ Runnable resumePredictionExecution = predictionTestControl.pauseExecution();
+ predictionTestControl.synchronizeNextSubmissions(2);
+ ExecutorService submissions = Executors.newFixedThreadPool(2);
+ try {
+ List> results =
+ Arrays.asList(
+ submissions.submit(
+ () -> catalog.submitPrediction(fixture.source, firstSpec)),
+ submissions.submit(
+ () -> catalog.submitPrediction(fixture.source, secondSpec)));
+ int accepted = 0;
+ int conflicts = 0;
+ for (Future result : results) {
+ try {
+ result.get(10, TimeUnit.SECONDS);
+ accepted++;
+ } catch (ExecutionException e) {
+ assertThat(e.getCause()).isInstanceOf(AlreadyExistsException.class);
+ conflicts++;
+ }
+ }
+ assertThat(accepted).isEqualTo(1);
+ assertThat(conflicts).isEqualTo(1);
+ } finally {
+ resumePredictionExecution.run();
+ submissions.shutdownNow();
+ }
+ }
+
+ @Test
+ void testCancelAfterOutputTableCreationDoesNotPublishTable() throws Exception {
+ PredictionFixture fixture = createPredictionFixture("prediction_cancel_after_create");
+ Runnable resume = predictionTestControl.pauseAfterOutputCreated();
+ Prediction submitted =
+ catalog.submitPrediction(
+ fixture.source, fixture.spec("cancel-after-create", "cancel-create-token"));
+
+ predictionTestControl.awaitOutputCreated();
+ assertThat(catalog.cancelPrediction(submitted.predictionId()).status().state())
+ .isEqualTo(PredictionStatus.State.TERMINATING);
+ resume.run();
+
+ assertCanceledWithoutOutputTable(fixture, submitted.predictionId());
+ }
+
+ @Test
+ void testCancelAfterOutputCommitDoesNotPublishTable() throws Exception {
+ PredictionFixture fixture = createPredictionFixture("prediction_cancel_after_commit");
+ Runnable resume = predictionTestControl.pauseAfterOutputCommitted();
+ Prediction submitted =
+ catalog.submitPrediction(
+ fixture.source, fixture.spec("cancel-after-commit", "cancel-commit-token"));
+
+ predictionTestControl.awaitOutputCommitted();
+ assertThat(catalog.cancelPrediction(submitted.predictionId()).status().state())
+ .isEqualTo(PredictionStatus.State.TERMINATING);
+ resume.run();
+
+ assertCanceledWithoutOutputTable(fixture, submitted.predictionId());
+ }
+
+ @Test
+ void testPredictionDerivesMergeKeyFromPrimaryKey() throws Exception {
+ PredictionFixture fixture = createPredictionFixture("prediction_derived_merge_key");
+ Prediction submitted =
+ catalog.submitPrediction(
+ fixture.source,
+ fixture.specWithoutMergeKey("derived-merge-key", "derived-key-token"));
+
+ Prediction completed = waitForPrediction(submitted.predictionId());
+ assertThat(completed.status().terminationDetails().code())
+ .isEqualTo(PredictionStatus.TerminationCode.SUCCESS);
+ assertThat(batchRead(catalog.getTable(completed.output().table().toIdentifier())))
+ .containsExactlyInAnyOrder(
+ "+I[1, {\"text\":\"first\"}, null]", "+I[2, {\"text\":\"second\"}, null]");
+ }
+
+ @Test
+ void testPredictionRejectsDerivedMergeKeyUsingReservedOutputName() throws Exception {
+ PredictionFixture fixture =
+ createPredictionFixture("prediction_reserved_derived_key", "response");
+
+ assertThatThrownBy(
+ () ->
+ catalog.submitPrediction(
+ fixture.source,
+ fixture.specWithoutMergeKey(
+ "reserved-derived-key", "reserved-key-token")))
+ .isInstanceOf(BadRequestException.class)
+ .hasMessageContaining("reserved output name");
+ }
+
+ @Test
+ void testPredictionRejectsUnsupportedScanOptions() throws Exception {
+ PredictionFixture fixture = createPredictionFixture("prediction_scan_options", "id", 2);
+ for (Map.Entry option :
+ Arrays.asList(
+ new AbstractMap.SimpleEntry<>(CoreOptions.SCAN_BUCKET.key(), "0"),
+ new AbstractMap.SimpleEntry<>(
+ CoreOptions.STREAM_SCAN_MODE.key(), "file-monitor"))) {
+ PredictionSpec spec =
+ new PredictionSpec(
+ "restricted-" + option.getKey(),
+ fixture.model,
+ Collections.singletonMap("text", "content"),
+ Collections.singletonList("id"),
+ "Echo the input",
+ null,
+ Collections.singletonMap(option.getKey(), option.getValue()),
+ "restricted-" + option.getKey() + "-token");
+
+ assertThatThrownBy(() -> catalog.submitPrediction(fixture.source, spec))
+ .isInstanceOf(BadRequestException.class)
+ .hasMessageContaining(option.getKey());
+ }
+
+ PredictionFixture configured =
+ createPredictionFixture(
+ "prediction_stream_scan_mode",
+ "id",
+ 1,
+ Collections.singletonMap(
+ CoreOptions.STREAM_SCAN_MODE.key(), "file-monitor"));
+ assertThatThrownBy(
+ () ->
+ catalog.submitPrediction(
+ configured.source,
+ configured.spec(
+ "configured-stream-scan-mode",
+ "configured-stream-scan-mode-token")))
+ .isInstanceOf(BadRequestException.class)
+ .hasMessageContaining(CoreOptions.STREAM_SCAN_MODE.key());
+ }
+
+ @Test
+ void testPredictionPublishesEmptyResultTable() throws Exception {
+ PredictionFixture fixture = createPredictionFixture("prediction_empty_result");
+ FileStoreTable source = (FileStoreTable) catalog.getTable(fixture.source);
+ commitRows(
+ source,
+ GenericRow.ofKind(RowKind.DELETE, 1, BinaryString.fromString("first")),
+ GenericRow.ofKind(RowKind.DELETE, 2, BinaryString.fromString("second")));
+ assertThat(batchRead(source)).isEmpty();
+
+ Prediction submitted =
+ catalog.submitPrediction(
+ fixture.source, fixture.spec("empty-result", "empty-result-token"));
+ Prediction completed = waitForPrediction(submitted.predictionId());
+
+ assertThat(completed.status().terminationDetails().code())
+ .isEqualTo(PredictionStatus.TerminationCode.SUCCESS);
+ FileStoreTable output =
+ (FileStoreTable) catalog.getTable(completed.output().table().toIdentifier());
+ assertThat(output.snapshotManager().latestSnapshot()).isNotNull();
+ assertThat(output.snapshotManager().latestSnapshot().totalRecordCount()).isZero();
+ assertThat(batchRead(output)).isEmpty();
+ }
+
+ private PredictionFixture createPredictionFixture(String database) throws Exception {
+ return createPredictionFixture(database, "id");
+ }
+
+ private PredictionFixture createPredictionFixture(String database, String primaryKey)
+ throws Exception {
+ return createPredictionFixture(database, primaryKey, 1);
+ }
+
+ private PredictionFixture createPredictionFixture(
+ String database, String primaryKey, int buckets) throws Exception {
+ return createPredictionFixture(database, primaryKey, buckets, Collections.emptyMap());
+ }
+
+ private PredictionFixture createPredictionFixture(
+ String database, String primaryKey, int buckets, Map tableOptions)
+ throws Exception {
+ Identifier source = Identifier.create(database, "clips");
+ Identifier model = Identifier.create(database, "echo_model");
+ catalog.createDatabase(database, false);
+ Schema.Builder schema =
+ Schema.newBuilder()
+ .column(primaryKey, DataTypes.INT())
+ .column("content", DataTypes.STRING())
+ .primaryKey(primaryKey)
+ .option(CoreOptions.BUCKET.key(), Integer.toString(buckets));
+ tableOptions.forEach(schema::option);
+ catalog.createTable(source, schema.build(), false);
+ commitRows(
+ (FileStoreTable) catalog.getTable(source),
+ GenericRow.of(1, BinaryString.fromString("first")),
+ GenericRow.of(2, BinaryString.fromString("second")));
+ ProviderModelDefinition definition =
+ new ProviderModelDefinition(
+ "reference",
+ "echo",
+ Collections.singletonList(new ModelField("text", "STRING", null)),
+ Collections.singletonList(new ModelField("response", "STRING", null)),
+ Collections.emptyMap());
+ catalog.createModel(
+ model, new Model(model, "Deterministic reference model", definition), false);
+ return new PredictionFixture(source, model);
+ }
+
+ private void assertCanceledWithoutOutputTable(PredictionFixture fixture, String predictionId)
+ throws Exception {
+ Prediction canceled = waitForPrediction(predictionId);
+ assertThat(canceled.status().terminationDetails().code())
+ .isEqualTo(PredictionStatus.TerminationCode.CANCELED);
+ assertThat(canceled.output()).isNull();
+ String outputName = fixture.source.getObjectName() + "__" + predictionId.replace('-', '_');
+ assertThat(catalog.listTables(fixture.source.getDatabaseName())).doesNotContain(outputName);
+ }
+
+ private static final class PredictionFixture {
+
+ private final Identifier source;
+ private final Identifier model;
+
+ private PredictionFixture(Identifier source, Identifier model) {
+ this.source = source;
+ this.model = model;
+ }
+
+ private PredictionSpec spec(String name, String idempotencyToken) {
+ return spec(name, Collections.singletonList("id"), idempotencyToken);
+ }
+
+ private PredictionSpec specWithoutMergeKey(String name, String idempotencyToken) {
+ return spec(name, null, idempotencyToken);
+ }
+
+ private PredictionSpec spec(
+ String name, List mergeKeyColumns, String idempotencyToken) {
+ return new PredictionSpec(
+ name,
+ model,
+ Collections.singletonMap("text", "content"),
+ mergeKeyColumns,
+ "Echo the input",
+ null,
+ Collections.emptyMap(),
+ idempotencyToken);
+ }
+ }
+
+ private Prediction waitForPrediction(String predictionId) throws Exception {
+ long deadline = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(10);
+ while (System.currentTimeMillis() < deadline) {
+ Prediction prediction = catalog.getPrediction(predictionId);
+ if (prediction.status().isTerminal()) {
+ return prediction;
+ }
+ Thread.sleep(10L);
+ }
+ throw new AssertionError("Prediction did not terminate: " + predictionId);
+ }
+
+ private void waitForPredictionState(String predictionId, PredictionStatus.State expectedState)
+ throws Exception {
+ long deadline = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(10);
+ while (System.currentTimeMillis() < deadline) {
+ if (catalog.getPrediction(predictionId).status().state() == expectedState) {
+ return;
+ }
+ Thread.sleep(10L);
+ }
+ throw new AssertionError("Prediction did not reach " + expectedState + ": " + predictionId);
+ }
+
@Test
public void testAlterTableRethrowsForeignTypedNotFound() throws Exception {
// a 404 typed other than TABLE/COLUMN must not be swallowed into reported success
@@ -2002,6 +2386,7 @@ private RESTCatalog initCatalogUtil(
this.config = new ConfigResponse(defaultConf, ImmutableMap.of());
restCatalogServer =
new RESTCatalogServer(dataPath, this.authProvider, this.config, restWarehouse);
+ predictionTestControl = restCatalogServer.predictionTestControl();
restCatalogServer.start();
for (Map.Entry entry : this.authMap.entrySet()) {
options.set(entry.getKey(), entry.getValue());
diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogModelHandler.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogModelHandler.java
index 2ab0b27a7d82..afdd42f8455b 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogModelHandler.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogModelHandler.java
@@ -31,6 +31,8 @@
import okhttp3.mockwebserver.MockResponse;
+import javax.annotation.Nullable;
+
import java.util.Comparator;
import java.util.List;
import java.util.Map;
@@ -103,6 +105,11 @@ void removeDatabase(String databaseName) {
.equals(databaseName));
}
+ @Nullable
+ Model getModel(Identifier identifier) {
+ return modelStore.get(identifier.getFullName());
+ }
+
private MockResponse list(String databaseName, Map parameters) {
int maxResults = maxResults(parameters);
String pageToken = parameters.get(PAGE_TOKEN);
diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java
new file mode 100644
index 000000000000..d1e8a860501d
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java
@@ -0,0 +1,804 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.rest;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.Snapshot;
+import org.apache.paimon.catalog.Database;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.catalog.Prediction;
+import org.apache.paimon.catalog.PredictionOutput;
+import org.apache.paimon.catalog.PredictionSource;
+import org.apache.paimon.catalog.PredictionSpec;
+import org.apache.paimon.catalog.PredictionStatus;
+import org.apache.paimon.catalog.TableMetadata;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.reader.RecordReaderIterator;
+import org.apache.paimon.rest.requests.SubmitPredictionRequest;
+import org.apache.paimon.rest.responses.ErrorResponse;
+import org.apache.paimon.rest.responses.GetPredictionResponse;
+import org.apache.paimon.rest.responses.ListPredictionsResponse;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.schema.TableSchema;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.BatchTableCommit;
+import org.apache.paimon.table.sink.BatchTableWrite;
+import org.apache.paimon.table.sink.BatchWriteBuilder;
+import org.apache.paimon.table.source.ReadBuilder;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DataTypes;
+import org.apache.paimon.utils.InternalRowUtils;
+
+import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
+
+import okhttp3.mockwebserver.MockResponse;
+
+import javax.annotation.Nullable;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+
+import static org.apache.paimon.rest.RESTApi.MAX_RESULTS;
+import static org.apache.paimon.rest.RESTApi.PAGE_TOKEN;
+import static org.apache.paimon.rest.RESTCatalogServerUtils.parseRequest;
+
+/** Executable prediction endpoints for the local REST catalog reference integration. */
+final class RESTCatalogPredictionHandler implements AutoCloseable {
+
+ private static final String DATABASE = "database";
+ private static final String TABLE = "table";
+ private static final String NAME = "name";
+ private static final String STATE = "state";
+
+ private final RESTCatalogServer server;
+ private final RESTFileSystemCatalog catalog;
+ private final RESTCatalogModelHandler modelHandler;
+ private final Map databaseStore;
+ private final Map tableMetadataStore;
+ private final String predictionsPath;
+ private final Map jobs = new ConcurrentHashMap<>();
+ private final Map jobsByIdempotencyToken = new ConcurrentHashMap<>();
+ private final AtomicLong nextId = new AtomicLong();
+ private final ExecutorService executor;
+
+ private volatile CountDownLatch executionGate = new CountDownLatch(0);
+ private volatile ExecutionGate outputCreatedGate = ExecutionGate.open();
+ private volatile ExecutionGate outputCommittedGate = ExecutionGate.open();
+ private volatile @Nullable CyclicBarrier submissionBarrier;
+
+ RESTCatalogPredictionHandler(
+ RESTCatalogServer server,
+ RESTFileSystemCatalog catalog,
+ RESTCatalogModelHandler modelHandler,
+ Map databaseStore,
+ Map tableMetadataStore,
+ String predictionsPath) {
+ this.server = server;
+ this.catalog = catalog;
+ this.modelHandler = modelHandler;
+ this.databaseStore = databaseStore;
+ this.tableMetadataStore = tableMetadataStore;
+ this.predictionsPath = predictionsPath;
+ this.executor =
+ Executors.newSingleThreadExecutor(
+ runnable -> {
+ Thread thread =
+ new Thread(runnable, "paimon-reference-prediction-executor");
+ thread.setDaemon(true);
+ return thread;
+ });
+ }
+
+ MockResponse handle(
+ String method, String resourcePath, String data, Map parameters)
+ throws Exception {
+ if (predictionsPath.equals(resourcePath)) {
+ if ("POST".equals(method)) {
+ return submit(data);
+ } else if ("GET".equals(method)) {
+ return list(parameters);
+ }
+ return new MockResponse().setResponseCode(404);
+ }
+
+ String suffix = resourcePath.substring((predictionsPath + "/").length());
+ String[] segments = suffix.split("/");
+ String predictionId = RESTUtil.decodeString(segments[0]);
+ if (segments.length == 1 && "GET".equals(method)) {
+ return get(predictionId);
+ }
+ if (segments.length == 2 && "cancel".equals(segments[1]) && "POST".equals(method)) {
+ return cancel(predictionId);
+ }
+ return new MockResponse().setResponseCode(404);
+ }
+
+ void pauseExecution() {
+ executionGate = new CountDownLatch(1);
+ }
+
+ void resumeExecution() {
+ executionGate.countDown();
+ }
+
+ void synchronizeNextSubmissions(int parties) {
+ submissionBarrier = new CyclicBarrier(parties);
+ }
+
+ void pauseAfterOutputCreated() {
+ outputCreatedGate = ExecutionGate.paused();
+ }
+
+ void awaitOutputCreated() throws InterruptedException {
+ outputCreatedGate.awaitReached();
+ }
+
+ void resumeAfterOutputCreated() {
+ outputCreatedGate.resume();
+ }
+
+ void pauseAfterOutputCommitted() {
+ outputCommittedGate = ExecutionGate.paused();
+ }
+
+ void awaitOutputCommitted() throws InterruptedException {
+ outputCommittedGate.awaitReached();
+ }
+
+ void resumeAfterOutputCommitted() {
+ outputCommittedGate.resume();
+ }
+
+ private MockResponse submit(String data) throws Exception {
+ SubmitPredictionRequest request = parseRequest(data, SubmitPredictionRequest.class);
+ PredictionJob existing = jobsByIdempotencyToken.get(request.getIdempotencyToken());
+ if (existing != null) {
+ return idempotencyResponse(existing, data);
+ }
+
+ Identifier source = request.getSource().toIdentifier();
+ PredictionSpec spec = request.toPredictionSpec();
+ MockResponse validationError = validate(source, spec);
+ if (validationError != null) {
+ return validationError;
+ }
+
+ FileStoreTable sourceTable = (FileStoreTable) catalog.getTable(source);
+ long snapshotId = resolveSnapshot(sourceTable, spec);
+ String predictionId = String.format("prediction-%020d", nextId.incrementAndGet());
+ PredictionJob job =
+ new PredictionJob(
+ predictionId, request.getSource(), spec, snapshotId, data, executionGate);
+ awaitSubmissionBarrier();
+ PredictionJob selected =
+ jobsByIdempotencyToken.compute(
+ request.getIdempotencyToken(),
+ (token, current) -> {
+ if (current != null) {
+ return current;
+ }
+ jobs.put(predictionId, job);
+ try {
+ executor.submit(() -> execute(job));
+ return job;
+ } catch (RuntimeException e) {
+ jobs.remove(predictionId, job);
+ throw e;
+ }
+ });
+ return selected == job
+ ? response(new GetPredictionResponse(job.snapshot()), 202)
+ : idempotencyResponse(selected, data);
+ }
+
+ private MockResponse idempotencyResponse(PredictionJob existing, String data) {
+ if (existing.requestPayload.equals(data)) {
+ return response(new GetPredictionResponse(existing.snapshot()), 202);
+ }
+ return error(
+ ErrorResponse.RESOURCE_TYPE_PREDICTION,
+ existing.predictionId,
+ "idempotencyToken is already used by another prediction",
+ 409);
+ }
+
+ private void awaitSubmissionBarrier() throws Exception {
+ CyclicBarrier barrier = submissionBarrier;
+ if (barrier != null) {
+ barrier.await(10, TimeUnit.SECONDS);
+ submissionBarrier = null;
+ }
+ }
+
+ @Nullable
+ private MockResponse validate(Identifier source, PredictionSpec spec) {
+ if (!databaseStore.containsKey(source.getDatabaseName())) {
+ return error(
+ ErrorResponse.RESOURCE_TYPE_DATABASE,
+ source.getDatabaseName(),
+ "Database does not exist",
+ 404);
+ }
+ if (!tableMetadataStore.containsKey(source.getFullName())) {
+ return error(
+ ErrorResponse.RESOURCE_TYPE_TABLE,
+ source.getObjectName(),
+ "Table does not exist",
+ 404);
+ }
+ Identifier model = spec.model();
+ if (!databaseStore.containsKey(model.getDatabaseName())) {
+ return error(
+ ErrorResponse.RESOURCE_TYPE_DATABASE,
+ model.getDatabaseName(),
+ "Database does not exist",
+ 404);
+ }
+ if (modelHandler.getModel(model) == null) {
+ return error(
+ ErrorResponse.RESOURCE_TYPE_MODEL,
+ model.getObjectName(),
+ "Model does not exist",
+ 404);
+ }
+ if (spec.where() != null) {
+ return error(null, null, "The reference provider does not evaluate where", 501);
+ }
+
+ TableSchema schema;
+ try {
+ schema = ((FileStoreTable) catalog.getTable(source)).schema();
+ } catch (Exception e) {
+ return error(
+ ErrorResponse.RESOURCE_TYPE_TABLE,
+ source.getObjectName(),
+ "Table does not exist",
+ 404);
+ }
+ List fields = schema.fieldNames();
+ List unsupportedScanOptions =
+ unsupportedScanOptions(schema.options(), spec.options());
+ if (!unsupportedScanOptions.isEmpty()) {
+ return error(
+ null,
+ null,
+ "Prediction does not support scan options: " + unsupportedScanOptions,
+ 400);
+ }
+ for (String input : spec.inputs().values()) {
+ if (!fields.contains(input)) {
+ return error(
+ ErrorResponse.RESOURCE_TYPE_COLUMN,
+ input,
+ "Prediction input column does not exist",
+ 404);
+ }
+ }
+ for (String key : mergeKeyColumns(schema, spec)) {
+ if (!fields.contains(key)) {
+ return error(
+ ErrorResponse.RESOURCE_TYPE_COLUMN,
+ key,
+ "Prediction merge key column does not exist",
+ 404);
+ }
+ if ("response".equals(key) || "error".equals(key)) {
+ return error(null, null, "mergeKeyColumns use a reserved output name", 400);
+ }
+ }
+ return null;
+ }
+
+ private long resolveSnapshot(FileStoreTable table, PredictionSpec spec) {
+ CoreOptions options = new CoreOptions(spec.options());
+ Long snapshotId = options.scanSnapshotId();
+ if (snapshotId != null) {
+ Snapshot snapshot = table.snapshotManager().snapshot(snapshotId);
+ if (snapshot == null) {
+ throw new IllegalArgumentException("Snapshot " + snapshotId + " does not exist");
+ }
+ return snapshot.id();
+ }
+ String tagName = options.scanTagName();
+ if (tagName != null) {
+ return table.tagManager().getOrThrow(tagName).id();
+ }
+ Snapshot latest = table.snapshotManager().latestSnapshot();
+ if (latest == null) {
+ throw new IllegalArgumentException("The prediction source has no snapshot");
+ }
+ return latest.id();
+ }
+
+ private void execute(PredictionJob job) {
+ @Nullable Identifier preparedOutput = null;
+ try {
+ if (!job.markRunning()) {
+ job.completeCanceled();
+ return;
+ }
+ job.executionGate.await();
+ checkCanceled(job);
+ preparedOutput = executePrediction(job);
+ Identifier output = preparedOutput;
+ if (!job.publishSuccess(output, () -> publishOutput(output))) {
+ cleanupOutput(output);
+ job.completeCanceled();
+ }
+ } catch (InterruptedException e) {
+ cleanupOutput(job.stagingOutput());
+ job.completeCanceled();
+ Thread.currentThread().interrupt();
+ } catch (Throwable t) {
+ cleanupOutput(preparedOutput == null ? job.stagingOutput() : preparedOutput);
+ if (job.isTerminating()) {
+ job.completeCanceled();
+ } else {
+ job.completeFailure(
+ t.getMessage() == null ? t.getClass().getSimpleName() : t.getMessage());
+ }
+ }
+ }
+
+ private Identifier executePrediction(PredictionJob job) throws Exception {
+ Identifier sourceIdentifier = job.source.toIdentifier();
+ FileStoreTable source = (FileStoreTable) catalog.getTable(sourceIdentifier);
+ Map scanOptions = new HashMap<>(job.spec.options());
+ scanOptions.remove(CoreOptions.SCAN_TAG_NAME.key());
+ scanOptions.put(CoreOptions.SCAN_SNAPSHOT_ID.key(), Long.toString(job.snapshotId));
+ source = source.copy(scanOptions);
+
+ Identifier outputIdentifier =
+ Identifier.create(
+ sourceIdentifier.getDatabaseName(),
+ sourceIdentifier.getObjectName()
+ + "__"
+ + job.predictionId.replace('-', '_'));
+ List mergeKeyColumns = mergeKeyColumns(source.schema(), job.spec);
+ Schema outputSchema = outputSchema(source.schema(), mergeKeyColumns);
+ catalog.createTable(outputIdentifier, outputSchema, false);
+ job.setStagingOutput(outputIdentifier);
+ FileStoreTable output = (FileStoreTable) catalog.getTable(outputIdentifier);
+ outputCreatedGate.reachAndAwait();
+ checkCanceled(job);
+
+ Map fieldIndexes = fieldIndexes(source.schema());
+ BatchWriteBuilder writeBuilder = output.newBatchWriteBuilder();
+ ReadBuilder readBuilder = source.newReadBuilder();
+ try (RecordReaderIterator rows =
+ new RecordReaderIterator<>(
+ readBuilder
+ .newRead()
+ .createReader(readBuilder.newScan().plan().splits()));
+ BatchTableWrite write = writeBuilder.newWrite();
+ BatchTableCommit commit = writeBuilder.newCommit()) {
+ while (rows.hasNext()) {
+ checkCanceled(job);
+ InternalRow sourceRow = rows.next();
+ write.write(
+ toOutputRow(
+ sourceRow,
+ source.schema(),
+ fieldIndexes,
+ mergeKeyColumns,
+ job.spec));
+ }
+ checkCanceled(job);
+ commit.commit(write.prepareCommit());
+ }
+
+ outputCommittedGate.reachAndAwait();
+ checkCanceled(job);
+ return outputIdentifier;
+ }
+
+ private void publishOutput(Identifier identifier) {
+ try {
+ FileStoreTable table = (FileStoreTable) catalog.getTable(identifier);
+ Snapshot snapshot = table.snapshotManager().latestSnapshot();
+ registerOutputTable(identifier, table);
+ server.setTableSnapshot(
+ identifier,
+ snapshot,
+ snapshot.totalRecordCount(),
+ 0L,
+ 0L,
+ snapshot.timeMillis());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void cleanupOutput(@Nullable Identifier identifier) {
+ if (identifier == null) {
+ return;
+ }
+ tableMetadataStore.remove(identifier.getFullName());
+ try {
+ catalog.dropTable(identifier, true);
+ } catch (Exception ignored) {
+ // Best-effort cleanup for a test reference provider.
+ }
+ }
+
+ private void registerOutputTable(Identifier identifier, FileStoreTable table) {
+ TableSchema schema = table.schema();
+ Map options = new HashMap<>(schema.options());
+ options.put(CoreOptions.PATH.key(), catalog.getTableLocation(identifier).toString());
+ tableMetadataStore.put(
+ identifier.getFullName(),
+ new TableMetadata(schema.copy(options), false, UUID.randomUUID().toString()));
+ }
+
+ private static Schema outputSchema(TableSchema source, List mergeKeyColumns) {
+ Schema.Builder builder =
+ Schema.newBuilder().option(CoreOptions.SNAPSHOT_IGNORE_EMPTY_COMMIT.key(), "false");
+ Map fields =
+ source.fields().stream().collect(Collectors.toMap(DataField::name, field -> field));
+ for (String name : mergeKeyColumns) {
+ builder.column(name, fields.get(name).type());
+ }
+ return builder.column("response", DataTypes.STRING())
+ .column("error", DataTypes.STRING())
+ .build();
+ }
+
+ private static List unsupportedScanOptions(
+ Map tableOptions, Map requestOptions) {
+ List unsupported = new ArrayList<>();
+ collectUnsupportedScanOptions(tableOptions, unsupported);
+ collectUnsupportedScanOptions(requestOptions, unsupported);
+ return unsupported;
+ }
+
+ private static void collectUnsupportedScanOptions(
+ Map options, List unsupported) {
+ for (String key : options.keySet()) {
+ if ((key.startsWith("scan.")
+ || key.startsWith("incremental-")
+ || CoreOptions.STREAM_SCAN_MODE.key().equals(key))
+ && !CoreOptions.SCAN_SNAPSHOT_ID.key().equals(key)
+ && !CoreOptions.SCAN_TAG_NAME.key().equals(key)
+ && !unsupported.contains(key)) {
+ unsupported.add(key);
+ }
+ }
+ }
+
+ private static List mergeKeyColumns(TableSchema source, PredictionSpec spec) {
+ return spec.mergeKeyColumns().isEmpty() ? source.primaryKeys() : spec.mergeKeyColumns();
+ }
+
+ private static Map fieldIndexes(TableSchema schema) {
+ Map indexes = new HashMap<>();
+ List fields = schema.fieldNames();
+ for (int i = 0; i < fields.size(); i++) {
+ indexes.put(fields.get(i), i);
+ }
+ return indexes;
+ }
+
+ private static InternalRow toOutputRow(
+ InternalRow sourceRow,
+ TableSchema sourceSchema,
+ Map fieldIndexes,
+ List mergeKeyColumns,
+ PredictionSpec spec)
+ throws JsonProcessingException {
+ Object[] values = new Object[mergeKeyColumns.size() + 2];
+ int outputIndex = 0;
+ for (String key : mergeKeyColumns) {
+ int sourceIndex = fieldIndexes.get(key);
+ DataType type = sourceSchema.fields().get(sourceIndex).type();
+ values[outputIndex++] =
+ InternalRowUtils.copy(InternalRowUtils.get(sourceRow, sourceIndex, type), type);
+ }
+
+ Map response = new LinkedHashMap<>();
+ for (Map.Entry input : spec.inputs().entrySet()) {
+ int sourceIndex = fieldIndexes.get(input.getValue());
+ DataType type = sourceSchema.fields().get(sourceIndex).type();
+ Object value = InternalRowUtils.get(sourceRow, sourceIndex, type);
+ response.put(input.getKey(), value == null ? null : value.toString());
+ }
+ values[outputIndex] = BinaryString.fromString(RESTApi.toJson(response));
+ values[outputIndex + 1] = null;
+ return GenericRow.of(values);
+ }
+
+ private MockResponse get(String predictionId) {
+ PredictionJob job = jobs.get(predictionId);
+ return job == null
+ ? notFound(predictionId)
+ : response(new GetPredictionResponse(job.snapshot()), 200);
+ }
+
+ private MockResponse cancel(String predictionId) {
+ PredictionJob job = jobs.get(predictionId);
+ if (job == null) {
+ return notFound(predictionId);
+ }
+ job.cancel();
+ return response(new GetPredictionResponse(job.snapshot()), 200);
+ }
+
+ private MockResponse list(Map parameters) {
+ List matching =
+ jobs.values().stream()
+ .filter(job -> matches(job, parameters))
+ .sorted(Comparator.comparing(PredictionJob::id).reversed())
+ .collect(Collectors.toList());
+ String pageToken = parameters.get(PAGE_TOKEN);
+ if (pageToken != null) {
+ matching =
+ matching.stream()
+ .filter(job -> job.id().compareTo(pageToken) < 0)
+ .collect(Collectors.toList());
+ }
+ int maxResults = maxResults(parameters);
+ boolean hasMore = matching.size() > maxResults;
+ List page = hasMore ? matching.subList(0, maxResults) : matching;
+ List responses =
+ page.stream()
+ .map(job -> new GetPredictionResponse(job.snapshot()))
+ .collect(Collectors.toList());
+ String nextPageToken = hasMore ? page.get(page.size() - 1).id() : null;
+ return response(new ListPredictionsResponse(responses, nextPageToken), 200);
+ }
+
+ private static boolean matches(PredictionJob job, Map parameters) {
+ Prediction prediction = job.snapshot();
+ return matches(parameters.get(DATABASE), prediction.source().database())
+ && matches(parameters.get(TABLE), prediction.source().table())
+ && matches(parameters.get(NAME), prediction.name())
+ && matches(parameters.get(STATE), prediction.status().state().name());
+ }
+
+ private static boolean matches(@Nullable String expected, @Nullable String actual) {
+ return expected == null || expected.equals(actual);
+ }
+
+ private static int maxResults(Map parameters) {
+ String value = parameters.get(MAX_RESULTS);
+ if (value == null) {
+ return RESTCatalogServer.DEFAULT_MAX_RESULTS;
+ }
+ int requested = Integer.parseInt(value);
+ return requested <= 0
+ ? RESTCatalogServer.DEFAULT_MAX_RESULTS
+ : Math.min(requested, RESTCatalogServer.DEFAULT_MAX_RESULTS);
+ }
+
+ private static void checkCanceled(PredictionJob job) throws InterruptedException {
+ if (job.isTerminating() || Thread.currentThread().isInterrupted()) {
+ throw new InterruptedException("Prediction was canceled");
+ }
+ }
+
+ private static MockResponse notFound(String predictionId) {
+ return error(
+ ErrorResponse.RESOURCE_TYPE_PREDICTION,
+ predictionId,
+ "Prediction does not exist",
+ 404);
+ }
+
+ private static MockResponse error(
+ @Nullable String resourceType,
+ @Nullable String resourceName,
+ String message,
+ int status) {
+ return response(new ErrorResponse(resourceType, resourceName, message, status), status);
+ }
+
+ private static MockResponse response(Object response, int status) {
+ try {
+ return new MockResponse()
+ .setResponseCode(status)
+ .setBody(RESTApi.toJson(response))
+ .addHeader("Content-Type", "application/json");
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void close() {
+ executionGate.countDown();
+ outputCreatedGate.resume();
+ outputCommittedGate.resume();
+ executor.shutdownNow();
+ try {
+ executor.awaitTermination(10, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ private static final class ExecutionGate {
+
+ private final CountDownLatch reached;
+ private final CountDownLatch resume;
+
+ private ExecutionGate(int count) {
+ this.reached = new CountDownLatch(count);
+ this.resume = new CountDownLatch(count);
+ }
+
+ private static ExecutionGate open() {
+ return new ExecutionGate(0);
+ }
+
+ private static ExecutionGate paused() {
+ return new ExecutionGate(1);
+ }
+
+ private void reachAndAwait() throws InterruptedException {
+ reached.countDown();
+ resume.await();
+ }
+
+ private void awaitReached() throws InterruptedException {
+ if (!reached.await(10, TimeUnit.SECONDS)) {
+ throw new AssertionError("Prediction execution did not reach the expected phase");
+ }
+ }
+
+ private void resume() {
+ resume.countDown();
+ }
+ }
+
+ private static final class PredictionJob {
+
+ private final String predictionId;
+ private final PredictionSource source;
+ private final PredictionSpec spec;
+ private final long snapshotId;
+ private final String requestPayload;
+ private final CountDownLatch executionGate;
+
+ private PredictionStatus status =
+ new PredictionStatus(PredictionStatus.State.PENDING, null);
+ private @Nullable PredictionOutput output;
+ private @Nullable Long startTime;
+ private @Nullable Long endTime;
+ private @Nullable Identifier stagingOutput;
+
+ private PredictionJob(
+ String predictionId,
+ PredictionSource source,
+ PredictionSpec spec,
+ long snapshotId,
+ String requestPayload,
+ CountDownLatch executionGate) {
+ this.predictionId = predictionId;
+ this.source = source;
+ this.spec = spec;
+ this.snapshotId = snapshotId;
+ this.requestPayload = requestPayload;
+ this.executionGate = executionGate;
+ }
+
+ private synchronized String id() {
+ return predictionId;
+ }
+
+ private synchronized boolean markRunning() {
+ if (status.state() != PredictionStatus.State.PENDING) {
+ return false;
+ }
+ status = new PredictionStatus(PredictionStatus.State.RUNNING, null);
+ startTime = System.currentTimeMillis();
+ return true;
+ }
+
+ private synchronized boolean publishSuccess(
+ Identifier outputIdentifier, Runnable publisher) {
+ if (status.state() != PredictionStatus.State.RUNNING) {
+ return false;
+ }
+ publisher.run();
+ output = new PredictionOutput(new PredictionSource(outputIdentifier));
+ status =
+ new PredictionStatus(
+ PredictionStatus.State.TERMINATED,
+ new PredictionStatus.TerminationDetails(
+ PredictionStatus.TerminationCode.SUCCESS, null));
+ endTime = System.currentTimeMillis();
+ return true;
+ }
+
+ private synchronized void completeFailure(String message) {
+ if (status.isTerminal()) {
+ return;
+ }
+ status =
+ new PredictionStatus(
+ PredictionStatus.State.TERMINATED,
+ new PredictionStatus.TerminationDetails(
+ PredictionStatus.TerminationCode.FAILED, message));
+ endTime = System.currentTimeMillis();
+ }
+
+ private synchronized void completeCanceled() {
+ if (status.isTerminal()) {
+ return;
+ }
+ status =
+ new PredictionStatus(
+ PredictionStatus.State.TERMINATED,
+ new PredictionStatus.TerminationDetails(
+ PredictionStatus.TerminationCode.CANCELED, null));
+ endTime = System.currentTimeMillis();
+ }
+
+ private synchronized void setStagingOutput(Identifier stagingOutput) {
+ this.stagingOutput = stagingOutput;
+ }
+
+ private synchronized @Nullable Identifier stagingOutput() {
+ return stagingOutput;
+ }
+
+ private synchronized boolean isTerminating() {
+ return status.state() == PredictionStatus.State.TERMINATING;
+ }
+
+ private synchronized void cancel() {
+ if (status.isTerminal() || status.state() == PredictionStatus.State.TERMINATING) {
+ return;
+ }
+ if (status.state() == PredictionStatus.State.PENDING) {
+ completeCanceled();
+ } else {
+ status = new PredictionStatus(PredictionStatus.State.TERMINATING, null);
+ }
+ }
+
+ private synchronized Prediction snapshot() {
+ return new Prediction(
+ predictionId,
+ source,
+ spec.name(),
+ status,
+ snapshotId,
+ spec.model(),
+ output,
+ startTime,
+ endTime);
+ }
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java
new file mode 100644
index 000000000000..67b0f165cbc4
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.rest;
+
+import org.apache.paimon.PagedList;
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.CatalogContext;
+import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.catalog.Prediction;
+import org.apache.paimon.catalog.PredictionSpec;
+import org.apache.paimon.catalog.PredictionStatus;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.rest.exceptions.NoSuchResourceException;
+
+import okhttp3.mockwebserver.MockResponse;
+import okhttp3.mockwebserver.MockWebServer;
+import okhttp3.mockwebserver.RecordedRequest;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** REST catalog tests for durable asynchronous predictions. */
+class RESTCatalogPredictionTest {
+
+ @Test
+ void testPredictionLifecycleUsesSharedRestContract() throws Exception {
+ try (MockWebServer server = new MockWebServer()) {
+ server.start();
+ Catalog catalog = catalog(server);
+ Identifier table = Identifier.create("quality", "clips");
+
+ server.enqueue(json(predictionJson("PENDING"), 202));
+ Prediction submitted = catalog.submitPrediction(table, spec());
+ assertThat(submitted.predictionId()).isEqualTo("prediction-1");
+ RecordedRequest submit = server.takeRequest(10, TimeUnit.SECONDS);
+ assertThat(submit.getPath()).isEqualTo("/v1/catalog/predictions");
+ Map request = RESTApi.fromJson(submit.getBody().readUtf8(), Map.class);
+ assertThat(request.get("idempotencyToken")).isInstanceOf(String.class);
+ assertThat((String) request.get("idempotencyToken")).isNotBlank();
+ assertThat(request)
+ .doesNotContainKeys("resultColumns", "mergeKeyColumns", "outputSchema");
+ assertThat((Map) request.get("source"))
+ .containsEntry("database", "quality")
+ .containsEntry("table", "clips");
+
+ server.enqueue(
+ json(
+ "{\"predictions\":["
+ + predictionJson("RUNNING")
+ + "],\"nextPageToken\":\"next\"}",
+ 200));
+ PagedList page =
+ catalog.listPredictions(
+ table, "daily-quality", PredictionStatus.State.RUNNING, 10, "page-1");
+ assertThat(page.getElements())
+ .extracting(Prediction::predictionId)
+ .containsExactly("prediction-1");
+ assertThat(page.getNextPageToken()).isEqualTo("next");
+ String listPath = server.takeRequest(10, TimeUnit.SECONDS).getPath();
+ assertThat(listPath)
+ .contains("database=quality")
+ .contains("table=clips")
+ .contains("name=daily-quality")
+ .contains("state=RUNNING")
+ .contains("maxResults=10")
+ .contains("pageToken=page-1");
+
+ server.enqueue(json(terminatedPredictionJson("CANCELED"), 200));
+ PredictionStatus cancelled = catalog.cancelPrediction("prediction/1").status();
+ assertThat(cancelled.state()).isEqualTo(PredictionStatus.State.TERMINATED);
+ assertThat(cancelled.terminationDetails().code())
+ .isEqualTo(PredictionStatus.TerminationCode.CANCELED);
+ assertThat(server.takeRequest(10, TimeUnit.SECONDS).getPath())
+ .isEqualTo("/v1/catalog/predictions/prediction%2F1/cancel");
+ }
+ }
+
+ @Test
+ void testPredictionNotFoundPreservesResourceType() throws Exception {
+ try (MockWebServer server = new MockWebServer()) {
+ server.start();
+ Catalog catalog = catalog(server);
+ Identifier table = Identifier.create("quality", "clips");
+
+ server.enqueue(notFound("DATABASE", "models"));
+ assertThatThrownBy(() -> catalog.submitPrediction(table, spec()))
+ .isInstanceOf(Catalog.DatabaseNotExistException.class)
+ .hasMessage("Database models does not exist.");
+
+ server.enqueue(notFound("MODEL", "qwen_vl"));
+ assertThatThrownBy(() -> catalog.submitPrediction(table, spec()))
+ .isInstanceOf(Catalog.ModelNotExistException.class);
+
+ server.enqueue(notFound("TABLE", "clips"));
+ assertThatThrownBy(() -> catalog.submitPrediction(table, spec()))
+ .isInstanceOf(Catalog.TableNotExistException.class);
+
+ server.enqueue(untypedNotFound());
+ assertThatThrownBy(() -> catalog.submitPrediction(table, spec()))
+ .isInstanceOf(NoSuchResourceException.class);
+
+ server.enqueue(notFound("PREDICTION", "prediction-1"));
+ assertThatThrownBy(() -> catalog.getPrediction("prediction-1"))
+ .isInstanceOf(Catalog.PredictionNotExistException.class);
+
+ server.enqueue(untypedNotFound());
+ assertThatThrownBy(() -> catalog.getPrediction("prediction-1"))
+ .isInstanceOf(NoSuchResourceException.class);
+
+ server.enqueue(untypedNotFound());
+ assertThatThrownBy(() -> catalog.cancelPrediction("prediction-1"))
+ .isInstanceOf(NoSuchResourceException.class);
+ }
+ }
+
+ @Test
+ void testPredictionRejectsBranchAndSystemTable() throws Exception {
+ try (MockWebServer server = new MockWebServer()) {
+ server.start();
+ Catalog catalog = catalog(server);
+
+ assertThatThrownBy(
+ () ->
+ catalog.submitPrediction(
+ new Identifier("quality", "clips", "dev"), spec()))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("branch");
+ assertThatThrownBy(
+ () ->
+ catalog.submitPrediction(
+ new Identifier("quality", "clips", null, "snapshots"),
+ spec()))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("system table");
+ }
+ }
+
+ private static PredictionSpec spec() {
+ return new PredictionSpec(
+ "daily-quality",
+ Identifier.create("quality", "qwen_vl"),
+ Collections.singletonMap("video", "content"),
+ null,
+ "Classify",
+ null,
+ Collections.emptyMap(),
+ null);
+ }
+
+ private static Catalog catalog(MockWebServer server) {
+ Options options = new Options();
+ options.set(RESTCatalogOptions.URI, server.url("/").toString());
+ options.set(RESTCatalogOptions.TOKEN_PROVIDER, "bear");
+ options.set(RESTCatalogOptions.TOKEN, "token");
+ options.set(RESTCatalogInternalOptions.PREFIX, "catalog");
+ return new RESTCatalog(CatalogContext.create(options), false);
+ }
+
+ private static String predictionJson(String status) {
+ return "{\"predictionId\":\"prediction-1\",\"source\":{\"database\":\"quality\","
+ + "\"table\":\"clips\"},\"name\":\"daily-quality\",\"status\":{\"state\":\""
+ + status
+ + "\"},\"snapshotId\":7,\"model\":{\"database\":\"quality\","
+ + "\"object\":\"qwen_vl\"}}";
+ }
+
+ private static String terminatedPredictionJson(String code) {
+ return "{\"predictionId\":\"prediction-1\",\"source\":{\"database\":\"quality\","
+ + "\"table\":\"clips\"},\"name\":\"daily-quality\","
+ + "\"status\":{\"state\":\"TERMINATED\",\"terminationDetails\":{\"code\":\""
+ + code
+ + "\"}},\"snapshotId\":7,\"model\":{\"database\":\"quality\","
+ + "\"object\":\"qwen_vl\"},"
+ + "\"output\":{\"table\":{"
+ + "\"database\":\"system\",\"table\":\"results\"}},"
+ + "\"startTime\":10,\"endTime\":20}";
+ }
+
+ private static MockResponse json(String body, int code) {
+ return new MockResponse()
+ .setResponseCode(code)
+ .setHeader("Content-Type", "application/json")
+ .setBody(body);
+ }
+
+ private static MockResponse notFound(String resourceType, String resourceName) {
+ return json(
+ "{\"resourceType\":\""
+ + resourceType
+ + "\",\"resourceName\":\""
+ + resourceName
+ + "\",\"message\":\"not found\",\"code\":404}",
+ 404);
+ }
+
+ private static MockResponse untypedNotFound() {
+ return json("{\"message\":\"route not found\",\"code\":404}", 404);
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTestControl.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTestControl.java
new file mode 100644
index 000000000000..3afc3ec25b29
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTestControl.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.rest;
+
+/** Controls prediction execution phases for deterministic REST integration tests. */
+final class RESTCatalogPredictionTestControl {
+
+ private final RESTCatalogPredictionHandler handler;
+
+ RESTCatalogPredictionTestControl(RESTCatalogPredictionHandler handler) {
+ this.handler = handler;
+ }
+
+ Runnable pauseExecution() {
+ handler.pauseExecution();
+ return handler::resumeExecution;
+ }
+
+ void synchronizeNextSubmissions(int parties) {
+ handler.synchronizeNextSubmissions(parties);
+ }
+
+ Runnable pauseAfterOutputCreated() {
+ handler.pauseAfterOutputCreated();
+ return handler::resumeAfterOutputCreated;
+ }
+
+ void awaitOutputCreated() throws InterruptedException {
+ handler.awaitOutputCreated();
+ }
+
+ Runnable pauseAfterOutputCommitted() {
+ handler.pauseAfterOutputCommitted();
+ return handler::resumeAfterOutputCommitted;
+ }
+
+ void awaitOutputCommitted() throws InterruptedException {
+ handler.awaitOutputCommitted();
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java
index 0009d1d0287c..847456a2a993 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java
@@ -262,6 +262,7 @@ public class RESTCatalogServer {
private final Map functionStore = new ConcurrentHashMap<>();
private final RESTCatalogModelHandler modelHandler =
new RESTCatalogModelHandler(databaseStore, databaseLifecycleLocks);
+ private final RESTCatalogPredictionHandler predictionHandler;
private final Map> columnAuthHandler = new HashMap<>();
private final Map> rowFilterAuthHandler = new HashMap<>();
private final Map> columnMaskingAuthHandler = new HashMap<>();
@@ -300,6 +301,14 @@ public RESTCatalogServer(
throw new UncheckedIOException(e);
}
this.catalog = new RESTFileSystemCatalog(fileIO, warehousePath, catalogContext);
+ this.predictionHandler =
+ new RESTCatalogPredictionHandler(
+ this,
+ catalog,
+ modelHandler,
+ databaseStore,
+ tableMetadataStore,
+ resourcePaths.predictions());
Dispatcher dispatcher = initDispatcher(authProvider);
MockWebServer mockWebServer = new MockWebServer();
mockWebServer.setDispatcher(dispatcher);
@@ -322,9 +331,14 @@ public void setTableSchema(Identifier identifier, TableSchema schema) {
}
public void shutdown() throws IOException {
+ predictionHandler.close();
server.shutdown();
}
+ RESTCatalogPredictionTestControl predictionTestControl() {
+ return new RESTCatalogPredictionTestControl(predictionHandler);
+ }
+
public void setTableSnapshot(
Identifier identifier,
Snapshot snapshot,
@@ -498,6 +512,10 @@ && isTableByIdRequest(request.getPath())) {
} else if (StringUtils.startsWith(
request.getPath(), resourcePaths.functions())) {
return functionsHandle(parameters);
+ } else if (resourcePaths.predictions().equals(resourcePath)
+ || resourcePath.startsWith(resourcePaths.predictions() + "/")) {
+ return predictionHandler.handle(
+ restAuthParameter.method(), resourcePath, data, parameters);
} else if (request.getPath().startsWith(databaseUri)) {
String[] resources =
resourcePath.substring((databaseUri + "/").length()).split("/");
diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/ResourcePathsTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/ResourcePathsTest.java
index b6f0f39f38f5..e31a2899a1b5 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/ResourcePathsTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/ResourcePathsTest.java
@@ -71,4 +71,13 @@ public void testPoliciesAreNestedUnderAttachmentResource() {
"/v1/catalog%2Fid/databases/sales+db/tables/orders%2Fall/policies/drop",
paths.dropPolicy(table));
}
+
+ @Test
+ public void testPredictionRejectsDotSegments() {
+ ResourcePaths paths = new ResourcePaths("paimon");
+ assertThrows(IllegalArgumentException.class, () -> paths.prediction("."));
+ assertThrows(IllegalArgumentException.class, () -> paths.prediction(".."));
+ assertThrows(IllegalArgumentException.class, () -> paths.cancelPrediction("."));
+ assertThrows(IllegalArgumentException.class, () -> paths.cancelPrediction(".."));
+ }
}
diff --git a/paimon-python/pypaimon/api/api_response.py b/paimon-python/pypaimon/api/api_response.py
index 67609852af66..5e45a9afb5c3 100644
--- a/paimon-python/pypaimon/api/api_response.py
+++ b/paimon-python/pypaimon/api/api_response.py
@@ -50,6 +50,7 @@ class ErrorResponse(RESTResponse):
RESOURCE_TYPE_VIEW = "VIEW"
RESOURCE_TYPE_FUNCTION = "FUNCTION"
RESOURCE_TYPE_MODEL = "MODEL"
+ RESOURCE_TYPE_PREDICTION = "PREDICTION"
RESOURCE_TYPE_COLUMN = "COLUMN"
RESOURCE_TYPE_SNAPSHOT = "SNAPSHOT"
RESOURCE_TYPE_TAG = "TAG"
diff --git a/paimon-python/pypaimon/api/prediction.py b/paimon-python/pypaimon/api/prediction.py
new file mode 100644
index 000000000000..036933677770
--- /dev/null
+++ b/paimon-python/pypaimon/api/prediction.py
@@ -0,0 +1,223 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import uuid
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+from pypaimon.api.api_request import RESTRequest
+from pypaimon.api.api_response import PagedResponse, RESTResponse
+from pypaimon.catalog.prediction import (
+ Prediction,
+ PredictionOutput as CatalogPredictionOutput,
+ PredictionSpec,
+ PredictionStatus,
+ _copy_string_list,
+ _copy_string_mapping,
+ _nonblank,
+)
+from pypaimon.common.json_util import json_field, optional_json_field
+from pypaimon.common.identifier import Identifier
+
+
+@dataclass
+class PredictionSource:
+ database: str
+ table: str
+
+ def __post_init__(self):
+ _nonblank(self.database, "database")
+ _nonblank(self.table, "table")
+
+ @classmethod
+ def from_identifier(cls, identifier: Identifier) -> "PredictionSource":
+ return cls(
+ identifier.get_database_name(), identifier.get_object_name())
+
+ def to_identifier(self) -> Identifier:
+ return Identifier.create(self.database, self.table)
+
+
+@dataclass
+class PredictionOutput:
+ table: PredictionSource
+
+ def __post_init__(self):
+ if not isinstance(self.table, PredictionSource):
+ raise ValueError("table must be a PredictionSource")
+
+ @classmethod
+ def from_dict(cls, data: Dict[str, Any]) -> "PredictionOutput":
+ table = data.get("table")
+ return cls(
+ table=(PredictionSource(**table)
+ if isinstance(table, dict) else table),
+ )
+
+ def to_catalog_output(self) -> CatalogPredictionOutput:
+ return CatalogPredictionOutput(table=self.table.to_identifier())
+
+
+@dataclass
+class SubmitPredictionRequest(RESTRequest):
+ source: PredictionSource
+ model: Identifier
+ inputs: Dict[str, str]
+ instructions: str
+ merge_key_columns: Optional[List[str]] = optional_json_field(
+ "mergeKeyColumns", "non_null")
+ name: Optional[str] = optional_json_field("name", "non_null")
+ idempotency_token: Optional[str] = optional_json_field(
+ "idempotencyToken", "non_null")
+ where: Optional[str] = optional_json_field("where", "non_null")
+ options: Dict[str, str] = field(default_factory=dict)
+
+ def __post_init__(self):
+ if not isinstance(self.model, Identifier):
+ raise ValueError("model must be an Identifier")
+ _nonblank(self.instructions, "instructions")
+ for value, field_name in (
+ (self.name, "name"),
+ (self.where, "where"),
+ ):
+ if value is not None:
+ _nonblank(value, field_name)
+ if self.idempotency_token is None:
+ self.idempotency_token = uuid.uuid4().hex
+ else:
+ _nonblank(self.idempotency_token, "idempotency_token")
+ if len(self.idempotency_token) > 64:
+ raise ValueError(
+ "idempotency_token must not exceed 64 characters")
+ self.inputs = _copy_string_mapping(self.inputs, "inputs", True)
+ if self.merge_key_columns is not None:
+ self.merge_key_columns = _copy_string_list(
+ self.merge_key_columns, "merge_key_columns")
+ self.options = _copy_string_mapping(self.options, "options", False)
+ PredictionSpec(
+ model=self.model,
+ inputs=self.inputs,
+ instructions=self.instructions,
+ merge_key_columns=self.merge_key_columns or [],
+ name=self.name,
+ where=self.where,
+ options=self.options,
+ )
+
+ @classmethod
+ def from_spec(
+ cls, source: Identifier, spec: PredictionSpec
+ ) -> "SubmitPredictionRequest":
+ return cls(
+ source=PredictionSource.from_identifier(source),
+ name=spec.name,
+ model=spec.model,
+ inputs=spec.inputs,
+ instructions=spec.instructions,
+ merge_key_columns=spec.merge_key_columns or None,
+ idempotency_token=spec.idempotency_token,
+ where=spec.where,
+ options=spec.options,
+ )
+
+
+@dataclass
+class CancelPredictionRequest(RESTRequest):
+ """Empty body for a prediction cancellation request."""
+
+ pass
+
+
+@dataclass
+class GetPredictionResponse(RESTResponse):
+ """Prediction metadata with UTC Unix epoch millisecond timestamps."""
+
+ prediction_id: str = json_field("predictionId")
+ source: PredictionSource = None
+ status: PredictionStatus = None
+ snapshot_id: int = json_field("snapshotId", default=0)
+ model: Identifier = None
+ name: Optional[str] = optional_json_field("name", "non_null")
+ output: Optional[PredictionOutput] = optional_json_field(
+ "output", "non_null")
+ start_time: Optional[int] = optional_json_field("startTime", "non_null")
+ end_time: Optional[int] = optional_json_field("endTime", "non_null")
+
+ @classmethod
+ def from_dict(cls, data: Dict[str, Any]) -> "GetPredictionResponse":
+ return cls(
+ prediction_id=data.get("predictionId"),
+ source=(
+ PredictionSource(**data.get("source"))
+ if isinstance(data.get("source"), dict)
+ else data.get("source")
+ ),
+ name=data.get("name"),
+ status=PredictionStatus.from_dict(data.get("status")),
+ snapshot_id=data.get("snapshotId"),
+ model=(
+ Identifier(**data.get("model"))
+ if isinstance(data.get("model"), dict)
+ else data.get("model")
+ ),
+ output=(PredictionOutput.from_dict(data.get("output"))
+ if isinstance(data.get("output"), dict)
+ else data.get("output")),
+ start_time=data.get("startTime"),
+ end_time=data.get("endTime"),
+ )
+
+ def to_prediction(self) -> Prediction:
+ return Prediction(
+ prediction_id=self.prediction_id,
+ source=self.source.to_identifier(),
+ name=self.name,
+ status=self.status,
+ snapshot_id=self.snapshot_id,
+ model=self.model,
+ output=(self.output.to_catalog_output()
+ if self.output is not None else None),
+ start_time=self.start_time,
+ end_time=self.end_time,
+ )
+
+
+@dataclass
+class ListPredictionsResponse(PagedResponse[GetPredictionResponse]):
+ predictions: List[GetPredictionResponse] = field(default_factory=list)
+ next_page_token: Optional[str] = json_field("nextPageToken", default=None)
+
+ def __post_init__(self):
+ if self.predictions is None:
+ self.predictions = []
+
+ def data(self) -> List[GetPredictionResponse]:
+ return self.predictions
+
+ def get_next_page_token(self) -> Optional[str]:
+ return self.next_page_token
+
+ @classmethod
+ def from_dict(cls, data: Dict[str, Any]) -> "ListPredictionsResponse":
+ return cls(
+ predictions=[
+ GetPredictionResponse.from_dict(value)
+ if isinstance(value, dict) else value
+ for value in (data.get("predictions") or [])
+ ],
+ next_page_token=data.get("nextPageToken"),
+ )
diff --git a/paimon-python/pypaimon/api/resource_paths.py b/paimon-python/pypaimon/api/resource_paths.py
index 807744132aee..7fa70f76c007 100644
--- a/paimon-python/pypaimon/api/resource_paths.py
+++ b/paimon-python/pypaimon/api/resource_paths.py
@@ -33,6 +33,7 @@ class ResourcePaths:
FUNCTIONS = "functions"
FUNCTION_DETAILS = "function-details"
MODELS = "models"
+ PREDICTIONS = "predictions"
TAGS = "tags"
BRANCHES = "branches"
FORWARD = "forward"
@@ -132,6 +133,20 @@ def model(self, database_name: str, model_name: str) -> str:
self.models(database_name), RESTUtil.encode_string(model_name)
)
+ def predictions(self) -> str:
+ return "{}/{}".format(self.base_path, self.PREDICTIONS)
+
+ def prediction(self, prediction_id: str) -> str:
+ if not isinstance(prediction_id, str) or not prediction_id.strip():
+ raise ValueError("prediction_id must be a non-empty string")
+ if prediction_id in (".", ".."):
+ raise ValueError("prediction_id must not be a dot segment")
+ return "{}/{}".format(
+ self.predictions(), RESTUtil.encode_string(prediction_id))
+
+ def cancel_prediction(self, prediction_id: str) -> str:
+ return "{}/cancel".format(self.prediction(prediction_id))
+
def tags(self, database_name: str, table_name: str) -> str:
return "{}/{}/{}/{}/{}/{}".format(
self.base_path, self.DATABASES, RESTUtil.encode_string(database_name),
diff --git a/paimon-python/pypaimon/api/rest_api.py b/paimon-python/pypaimon/api/rest_api.py
index cdd608f29cde..84d4910aeb23 100755
--- a/paimon-python/pypaimon/api/rest_api.py
+++ b/paimon-python/pypaimon/api/rest_api.py
@@ -53,6 +53,10 @@
from pypaimon.api.auth import AuthProviderFactory, RESTAuthFunction
from pypaimon.api.client import HttpClient
from pypaimon.api.model import CreateModelRequest, GetModelResponse, ListModelsResponse
+from pypaimon.api.prediction import (
+ CancelPredictionRequest, GetPredictionResponse, ListPredictionsResponse,
+ SubmitPredictionRequest,
+)
from pypaimon.api.resource_paths import ResourcePaths
from pypaimon.api.rest_exception import NoSuchResourceException
from pypaimon.api.rest_util import RESTUtil
@@ -61,6 +65,7 @@
from pypaimon.common.options import Options
from pypaimon.common.options.config import CatalogOptions
from pypaimon.common.identifier import Identifier
+from pypaimon.catalog.prediction import PredictionState
from pypaimon.management.data_policy import DataPolicy
from pypaimon.management.list_permissions_request import \
ListPermissionsRequest
@@ -785,6 +790,59 @@ def drop_model(self, identifier: Identifier) -> None:
self.rest_auth_function,
)
+ def submit_prediction(
+ self,
+ request: SubmitPredictionRequest,
+ ) -> GetPredictionResponse:
+ return self.client.post_with_response_type(
+ self.resource_paths.predictions(),
+ request,
+ GetPredictionResponse,
+ self.rest_auth_function,
+ )
+
+ def get_prediction(self, prediction_id: str) -> GetPredictionResponse:
+ return self.client.get(
+ self.resource_paths.prediction(prediction_id),
+ GetPredictionResponse,
+ self.rest_auth_function,
+ )
+
+ def list_predictions(
+ self,
+ source: Optional[Identifier] = None,
+ name: Optional[str] = None,
+ state: Optional[PredictionState] = None,
+ max_results: Optional[int] = None,
+ page_token: Optional[str] = None,
+ ) -> PagedList[GetPredictionResponse]:
+ query_params = {}
+ if source is not None:
+ database_name, table_name = self._validate_resource_path(source)
+ self._put_query_parameter(query_params, "database", database_name)
+ self._put_query_parameter(query_params, "table", table_name)
+ self._put_query_parameter(query_params, "name", name)
+ self._put_query_parameter(
+ query_params, "state", state.value if state is not None else None)
+ if max_results is not None and max_results > 0:
+ query_params[self.MAX_RESULTS] = str(max_results)
+ self._put_query_parameter(query_params, self.PAGE_TOKEN, page_token)
+ response = self.client.get_with_params(
+ self.resource_paths.predictions(),
+ query_params,
+ ListPredictionsResponse,
+ self.rest_auth_function,
+ )
+ return PagedList(response.predictions, response.next_page_token)
+
+ def cancel_prediction(self, prediction_id: str) -> GetPredictionResponse:
+ return self.client.post_with_response_type(
+ self.resource_paths.cancel_prediction(prediction_id),
+ CancelPredictionRequest(),
+ GetPredictionResponse,
+ self.rest_auth_function,
+ )
+
def auth_table_query(self, identifier: Identifier, select: Optional[List[str]]) -> AuthTableQueryResponse:
database_name, table_name = self.__validate_identifier(identifier)
request = AuthTableQueryRequest(select=select)
diff --git a/paimon-python/pypaimon/catalog/catalog.py b/paimon-python/pypaimon/catalog/catalog.py
index 9ea6a83074a0..0fab39ac6c4c 100644
--- a/paimon-python/pypaimon/catalog/catalog.py
+++ b/paimon-python/pypaimon/catalog/catalog.py
@@ -111,6 +111,43 @@ def drop_model(
"""
raise NotImplementedError("drop_model is not supported by this catalog.")
+ def submit_prediction(
+ self,
+ identifier: Union[str, Identifier],
+ spec: 'PredictionSpec',
+ ) -> 'Prediction':
+ """Submit an asynchronous prediction over a table snapshot."""
+ raise NotImplementedError(
+ "submit_prediction is not supported by this catalog.")
+
+ def get_prediction(
+ self,
+ prediction_id: str,
+ ) -> 'Prediction':
+ """Get one prediction by its unique id."""
+ raise NotImplementedError(
+ "get_prediction is not supported by this catalog.")
+
+ def list_predictions(
+ self,
+ source: Optional[Union[str, Identifier]] = None,
+ name: Optional[str] = None,
+ state: Optional['PredictionState'] = None,
+ max_results: Optional[int] = None,
+ page_token: Optional[str] = None,
+ ) -> PagedList['Prediction']:
+ """List predictions, optionally filtered by source table."""
+ raise NotImplementedError(
+ "list_predictions is not supported by this catalog.")
+
+ def cancel_prediction(
+ self,
+ prediction_id: str,
+ ) -> 'Prediction':
+ """Request cancellation and return the updated prediction."""
+ raise NotImplementedError(
+ "cancel_prediction is not supported by this catalog.")
+
def alter_database(self, name: str, changes: List[PropertyChange]) -> None:
"""Alter database properties.
diff --git a/paimon-python/pypaimon/catalog/catalog_exception.py b/paimon-python/pypaimon/catalog/catalog_exception.py
index cb2105401157..934e720d6dae 100644
--- a/paimon-python/pypaimon/catalog/catalog_exception.py
+++ b/paimon-python/pypaimon/catalog/catalog_exception.py
@@ -135,6 +135,14 @@ def __init__(self, identifier: Identifier):
super().__init__(f"Model {identifier.get_full_name()} already exists")
+class PredictionNotExistException(CatalogException):
+ """Prediction not exist exception"""
+
+ def __init__(self, prediction_id: str):
+ self.prediction_id = prediction_id
+ super().__init__(f"Prediction {prediction_id} does not exist")
+
+
class ColumnNotExistException(CatalogException):
"""Column not exist exception"""
diff --git a/paimon-python/pypaimon/catalog/prediction.py b/paimon-python/pypaimon/catalog/prediction.py
new file mode 100644
index 000000000000..1d86206a44e7
--- /dev/null
+++ b/paimon-python/pypaimon/catalog/prediction.py
@@ -0,0 +1,236 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Dict, List, Optional, Union
+
+from pypaimon.common.identifier import Identifier
+from pypaimon.common.json_util import optional_json_field
+from pypaimon.common.options.core_options import CoreOptions
+
+
+class PredictionState(str, Enum):
+ PENDING = "PENDING"
+ RUNNING = "RUNNING"
+ TERMINATING = "TERMINATING"
+ TERMINATED = "TERMINATED"
+
+
+class PredictionTerminationCode(str, Enum):
+ SUCCESS = "SUCCESS"
+ SUCCESS_WITH_FAILURES = "SUCCESS_WITH_FAILURES"
+ FAILED = "FAILED"
+ CANCELED = "CANCELED"
+
+
+@dataclass
+class PredictionTerminationDetails:
+ """Result and diagnostic message available after termination."""
+
+ code: PredictionTerminationCode
+ message: Optional[str] = None
+
+ def __post_init__(self):
+ if isinstance(self.code, str):
+ self.code = PredictionTerminationCode(self.code)
+ if not isinstance(self.code, PredictionTerminationCode):
+ raise ValueError("code must be a PredictionTerminationCode")
+
+
+@dataclass
+class PredictionStatus:
+ """Lifecycle and optional termination details of a prediction."""
+
+ state: PredictionState
+ termination_details: Optional[PredictionTerminationDetails] = \
+ optional_json_field("terminationDetails", "non_null")
+
+ def __post_init__(self):
+ if isinstance(self.state, str):
+ self.state = PredictionState(self.state)
+ if not isinstance(self.state, PredictionState):
+ raise ValueError("state must be a PredictionState")
+ if (self.state is PredictionState.TERMINATED
+ and self.termination_details is None):
+ raise ValueError(
+ "termination_details must be present for TERMINATED state")
+ accepts_details = self.state in {
+ PredictionState.TERMINATING,
+ PredictionState.TERMINATED,
+ }
+ if not accepts_details and self.termination_details is not None:
+ raise ValueError(
+ "termination_details must not be present before "
+ "termination starts")
+ if self.termination_details is not None and not isinstance(
+ self.termination_details, PredictionTerminationDetails):
+ raise ValueError(
+ "termination_details must be PredictionTerminationDetails")
+
+ @classmethod
+ def from_dict(cls, value):
+ details = value.get("terminationDetails")
+ return cls(
+ state=value.get("state"),
+ termination_details=(
+ PredictionTerminationDetails(**details)
+ if details is not None else None),
+ )
+
+ def to_dict(self):
+ result = {"state": self.state.value}
+ if self.termination_details is not None:
+ result["terminationDetails"] = {
+ "code": self.termination_details.code.value,
+ }
+ if self.termination_details.message is not None:
+ result["terminationDetails"]["message"] = \
+ self.termination_details.message
+ return result
+
+ @property
+ def terminal(self) -> bool:
+ return self.state is PredictionState.TERMINATED
+
+
+@dataclass
+class PredictionSpec:
+ """Provider-neutral input for an asynchronous table prediction."""
+
+ model: Union[str, Identifier]
+ inputs: Dict[str, str]
+ instructions: str
+ merge_key_columns: List[str] = field(default_factory=list)
+ name: Optional[str] = None
+ where: Optional[str] = None
+ options: Dict[str, str] = field(default_factory=dict)
+ idempotency_token: Optional[str] = None
+
+ def __post_init__(self):
+ if isinstance(self.model, str):
+ self.model = Identifier.from_string(self.model)
+ if not isinstance(self.model, Identifier):
+ raise ValueError("model must be an Identifier or qualified name")
+ _nonblank(self.model.get_database_name(), "model database")
+ _nonblank(self.model.get_object_name(), "model object")
+ _nonblank(self.instructions, "instructions")
+ for value, name in (
+ (self.name, "name"),
+ (self.where, "where"),
+ (self.idempotency_token, "idempotency_token"),
+ ):
+ if value is not None:
+ _nonblank(value, name)
+ if (self.idempotency_token is not None
+ and len(self.idempotency_token) > 64):
+ raise ValueError(
+ "idempotency_token must not exceed 64 characters")
+ self.inputs = _copy_string_mapping(self.inputs, "inputs", True)
+ self.merge_key_columns = _copy_string_list(
+ self.merge_key_columns, "merge_key_columns")
+ self.options = _copy_string_mapping(self.options, "options", False)
+ snapshot_id = self.options.get(CoreOptions.SCAN_SNAPSHOT_ID.key())
+ tag_name = self.options.get(CoreOptions.SCAN_TAG_NAME.key())
+ if snapshot_id is not None:
+ try:
+ if int(snapshot_id) <= 0:
+ raise ValueError
+ except ValueError:
+ raise ValueError(
+ "scan.snapshot-id must be a positive integer")
+ if snapshot_id is not None and tag_name is not None:
+ raise ValueError(
+ "scan.snapshot-id and scan.tag-name are mutually exclusive")
+
+
+@dataclass
+class PredictionOutput:
+ """Managed result table of an asynchronous prediction."""
+
+ table: Identifier
+
+ def __post_init__(self):
+ if not isinstance(self.table, Identifier):
+ raise ValueError("table must be an Identifier")
+
+
+@dataclass
+class Prediction:
+ """Durable metadata for one asynchronous prediction job.
+
+ ``start_time`` and ``end_time`` are Unix epoch milliseconds in UTC.
+ """
+
+ prediction_id: str
+ source: Identifier
+ status: PredictionStatus
+ snapshot_id: int
+ model: Union[str, Identifier]
+ name: Optional[str] = None
+ output: Optional[PredictionOutput] = None
+ start_time: Optional[int] = None
+ end_time: Optional[int] = None
+
+ def __post_init__(self):
+ _nonblank(self.prediction_id, "prediction_id")
+ if not isinstance(self.source, Identifier):
+ raise ValueError("source must be an Identifier")
+ if not isinstance(self.status, PredictionStatus):
+ raise ValueError("status must be a PredictionStatus")
+ if type(self.snapshot_id) is not int or self.snapshot_id <= 0:
+ raise ValueError("snapshot_id must be a positive integer")
+ if isinstance(self.model, str):
+ self.model = Identifier.from_string(self.model)
+ if not isinstance(self.model, Identifier):
+ raise ValueError("model must be an Identifier or qualified name")
+ if self.output is not None and not isinstance(
+ self.output, PredictionOutput):
+ raise ValueError("output must be a PredictionOutput")
+ for name in ("start_time", "end_time"):
+ value = getattr(self, name)
+ if value is not None and (type(value) is not int or value < 0):
+ raise ValueError(
+ "{} must be a non-negative integer".format(name))
+
+
+def _nonblank(value, name):
+ if not isinstance(value, str) or not value.strip():
+ raise ValueError("{} must be a non-empty string".format(name))
+
+
+def _copy_string_mapping(values, name, required):
+ if not isinstance(values, dict) or (required and not values):
+ raise ValueError("{} must be a non-empty mapping".format(name))
+ result = {}
+ for key, value in values.items():
+ _nonblank(key, "{} key".format(name))
+ _nonblank(value, "{} value".format(name))
+ result[key] = value
+ return result
+
+
+def _copy_string_list(values, name):
+ if not isinstance(values, list):
+ raise ValueError("{} must be a list".format(name))
+ result = []
+ for value in values:
+ _nonblank(value, "{} value".format(name))
+ if value in result:
+ raise ValueError("{} must not contain duplicates".format(name))
+ result.append(value)
+ return result
diff --git a/paimon-python/pypaimon/catalog/rest/rest_catalog.py b/paimon-python/pypaimon/catalog/rest/rest_catalog.py
index 6caf66564a05..2de8eca60d30 100644
--- a/paimon-python/pypaimon/catalog/rest/rest_catalog.py
+++ b/paimon-python/pypaimon/catalog/rest/rest_catalog.py
@@ -19,6 +19,7 @@
from typing import Any, Callable, Dict, List, Optional, Union
from pypaimon.api.api_response import ErrorResponse, GetTableResponse, GetTagResponse, PagedList, Partition
from pypaimon.api.model import CreateModelRequest
+from pypaimon.api.prediction import SubmitPredictionRequest
from pypaimon.api.rest_api import RESTApi
from pypaimon.api.rest_permission_management import RESTPermissionManagement
from pypaimon.api.rest_policy_management import RESTPolicyManagement
@@ -38,12 +39,14 @@
TableIdNotExistException, TableIdNoPermissionException,
FunctionNotExistException, FunctionAlreadyExistException,
ModelNotExistException, ModelAlreadyExistException,
+ PredictionNotExistException,
DefinitionAlreadyExistException, DefinitionNotExistException,
TagNotExistException, TagAlreadyExistException,
BranchNotExistException, BranchAlreadyExistException,
)
from pypaimon.catalog.database import Database
from pypaimon.catalog.model import Model, ModelDefinition
+from pypaimon.catalog.prediction import Prediction, PredictionSpec, PredictionState
from pypaimon.catalog.rest.property_change import PropertyChange
from pypaimon.catalog.rest.rest_token_file_io import RESTTokenFileIO
from pypaimon.catalog.rest.table_metadata import TableMetadata
@@ -678,6 +681,81 @@ def drop_model(
except NotImplementedException as e:
raise NotImplementedError("REST Catalog does not support models") from e
+ def submit_prediction(
+ self,
+ identifier: Union[str, Identifier],
+ spec: PredictionSpec,
+ ) -> Prediction:
+ if not isinstance(identifier, Identifier):
+ identifier = Identifier.from_string(identifier)
+ check_not_branch(identifier, "submit_prediction")
+ check_not_system_table(identifier, "submit_prediction")
+ try:
+ return self.rest_api.submit_prediction(
+ SubmitPredictionRequest.from_spec(identifier, spec)).to_prediction()
+ except NoSuchResourceException as e:
+ if e.resource_type == ErrorResponse.RESOURCE_TYPE_DATABASE:
+ raise DatabaseNotExistException(
+ e.resource_name
+ or identifier.get_database_name()) from e
+ if e.resource_type == ErrorResponse.RESOURCE_TYPE_MODEL:
+ raise ModelNotExistException(spec.model) from e
+ if e.resource_type == ErrorResponse.RESOURCE_TYPE_TABLE:
+ raise TableNotExistException(identifier) from e
+ raise
+ except NotImplementedException as e:
+ raise NotImplementedError(
+ "REST Catalog does not support predictions") from e
+
+ def get_prediction(
+ self,
+ prediction_id: str,
+ ) -> Prediction:
+ try:
+ return self.rest_api.get_prediction(prediction_id).to_prediction()
+ except NoSuchResourceException as e:
+ if e.resource_type == ErrorResponse.RESOURCE_TYPE_PREDICTION:
+ raise PredictionNotExistException(prediction_id) from e
+ raise
+ except NotImplementedException as e:
+ raise NotImplementedError(
+ "REST Catalog does not support predictions") from e
+
+ def list_predictions(
+ self,
+ source: Optional[Union[str, Identifier]] = None,
+ name: Optional[str] = None,
+ state: Optional[PredictionState] = None,
+ max_results: Optional[int] = None,
+ page_token: Optional[str] = None,
+ ) -> PagedList[Prediction]:
+ if source is not None and not isinstance(source, Identifier):
+ source = Identifier.from_string(source)
+ try:
+ page = self.rest_api.list_predictions(
+ source, name, state, max_results, page_token)
+ return PagedList(
+ [value.to_prediction() for value in page.elements],
+ page.next_page_token,
+ )
+ except NotImplementedException as e:
+ raise NotImplementedError(
+ "REST Catalog does not support predictions") from e
+
+ def cancel_prediction(
+ self,
+ prediction_id: str,
+ ) -> Prediction:
+ try:
+ return self.rest_api.cancel_prediction(prediction_id).to_prediction()
+ except NoSuchResourceException as e:
+ if e.resource_type == ErrorResponse.RESOURCE_TYPE_PREDICTION:
+ raise PredictionNotExistException(prediction_id) from e
+ raise
+ except NotImplementedException as e:
+ raise NotImplementedError(
+ "REST Catalog does not support predictions") from e
+
# Tag CRUD: mirrors Java RESTCatalog tag handlers.
def create_tag(self, identifier: Union[str, Identifier], tag_name: str,
snapshot_id: Optional[int] = None,
diff --git a/paimon-python/pypaimon/common/wait.py b/paimon-python/pypaimon/common/wait.py
new file mode 100644
index 000000000000..c23cefdd4d19
--- /dev/null
+++ b/paimon-python/pypaimon/common/wait.py
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from datetime import timedelta
+from typing import Callable, Generic, TypeVar
+
+
+T = TypeVar("T")
+
+
+class Wait(Generic[T]):
+ """Handle for a long-running operation."""
+
+ def __init__(self, waiter: Callable, response=None, **kwargs):
+ self.response = response
+ self._waiter = waiter
+ self._bind = kwargs
+
+ def __getattr__(self, name):
+ try:
+ return self._bind[name]
+ except KeyError:
+ raise AttributeError(name) from None
+
+ def bind(self):
+ return dict(self._bind)
+
+ def result(
+ self,
+ timeout: timedelta = timedelta(minutes=20),
+ callback: Callable[[T], None] = None,
+ ) -> T:
+ """Delegate waiting and return the final value.
+
+ The timeout is checked between polling requests. An in-flight request
+ may finish after the deadline, and timing out does not cancel the
+ server-side operation.
+ """
+ kwargs = self._bind.copy()
+ return self._waiter(callback=callback, timeout=timeout, **kwargs)
diff --git a/paimon-python/pypaimon/multimodal/prediction_wait.py b/paimon-python/pypaimon/multimodal/prediction_wait.py
new file mode 100644
index 000000000000..5f0d400edc92
--- /dev/null
+++ b/paimon-python/pypaimon/multimodal/prediction_wait.py
@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import random
+import time
+from datetime import timedelta
+from typing import Callable, Optional
+
+from pypaimon.catalog.prediction import Prediction
+from pypaimon.common.wait import Wait
+
+
+class PredictionWait(Wait[Prediction]):
+ """Wait handle returned after submitting an asynchronous prediction."""
+
+ def __init__(self, catalog, prediction: Prediction):
+ super().__init__(
+ lambda prediction_id, timeout, callback: _wait_for_prediction(
+ catalog, prediction_id, timeout, callback),
+ response=prediction,
+ prediction_id=prediction.prediction_id,
+ )
+
+
+def _wait_for_prediction(
+ catalog,
+ prediction_id: str,
+ timeout: timedelta,
+ callback: Optional[Callable[[Prediction], None]],
+) -> Prediction:
+ if not isinstance(timeout, timedelta):
+ raise ValueError("timeout must be a datetime.timedelta")
+ if timeout.total_seconds() <= 0:
+ raise ValueError("timeout must be positive")
+
+ deadline = time.monotonic() + timeout.total_seconds()
+ attempt = 1
+ while time.monotonic() < deadline:
+ prediction = catalog.get_prediction(prediction_id)
+ if time.monotonic() >= deadline:
+ break
+ if prediction.status.terminal:
+ return prediction
+ if callback is not None:
+ callback(prediction)
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ break
+ sleep_seconds = min(attempt, 10) + random.random()
+ time.sleep(min(sleep_seconds, remaining))
+ attempt += 1
+
+ raise TimeoutError(
+ "Prediction {} did not finish within {}".format(
+ prediction_id, timeout))
diff --git a/paimon-python/pypaimon/multimodal/table.py b/paimon-python/pypaimon/multimodal/table.py
index de39dcb5be20..cee8c5f474f1 100644
--- a/paimon-python/pypaimon/multimodal/table.py
+++ b/paimon-python/pypaimon/multimodal/table.py
@@ -111,6 +111,112 @@ def copy(self, options: dict) -> 'MultimodalTable':
return MultimodalTable(
self.catalog, raw_table.identifier.get_full_name(), raw_table)
+ def predict(
+ self,
+ *,
+ model,
+ inputs,
+ instructions,
+ merge_key_columns=None,
+ name=None,
+ where=None,
+ options=None,
+ idempotency_token=None):
+ """Submit an asynchronous prediction without reading rows locally.
+
+ ``options`` uses Paimon dynamic table option names. Time travel uses
+ ``scan.snapshot-id`` or ``scan.tag-name``.
+
+ The returned waiter exposes the prediction ID immediately. Calling
+ ``result()`` is optional and intended for interactive clients; durable
+ workflows should persist the ID and observe the prediction
+ asynchronously.
+ """
+ from pypaimon.catalog.prediction import PredictionSpec
+ from pypaimon.multimodal.prediction_wait import PredictionWait
+ from pypaimon.table.file_store_table import FileStoreTable
+
+ if not isinstance(self.raw_table, FileStoreTable):
+ raise NotImplementedError(
+ "Prediction requires a Paimon file store table")
+ table_options = self.raw_table.options.options.to_map()
+ allowed = {"scan.snapshot-id", "scan.tag-name"}
+ prediction_options = dict(options or {})
+
+ def unsupported_scan_option(key):
+ return key not in allowed and (
+ key.startswith("scan.")
+ or key.startswith("incremental-")
+ or key == "stream-scan-mode")
+
+ unsupported = [
+ key for key in table_options
+ if unsupported_scan_option(key)
+ ]
+ unsupported.extend([
+ key for key in prediction_options
+ if unsupported_scan_option(key)
+ ])
+ if unsupported:
+ raise ValueError(
+ "Prediction does not support scan options: {}".format(
+ unsupported))
+ for key in allowed:
+ table_value = table_options.get(key)
+ if table_value is None:
+ continue
+ request_value = prediction_options.get(key)
+ if request_value is not None and request_value != table_value:
+ raise ValueError(
+ "Conflicting prediction option: {}".format(key))
+ prediction_options[key] = table_value
+ spec = PredictionSpec(
+ name=name,
+ model=model,
+ inputs=inputs,
+ instructions=instructions,
+ merge_key_columns=(
+ [] if merge_key_columns is None else merge_key_columns),
+ where=where,
+ options=prediction_options,
+ idempotency_token=idempotency_token,
+ )
+ prediction = self.catalog.submit_prediction(
+ self.raw_table.identifier, spec)
+ return PredictionWait(self.catalog, prediction)
+
+ def get_prediction(self, prediction_id):
+ """Return metadata for one asynchronous prediction."""
+ return self.catalog.get_prediction(prediction_id)
+
+ def cancel_prediction(self, prediction_id):
+ """Request cancellation and return a wait handle."""
+ from pypaimon.multimodal.prediction_wait import PredictionWait
+
+ prediction = self.catalog.cancel_prediction(prediction_id)
+ return PredictionWait(self.catalog, prediction)
+
+ def list_predictions(
+ self,
+ *,
+ name=None,
+ state=None,
+ max_results=None,
+ page_token=None):
+ """List prediction metadata for this table."""
+ from pypaimon.catalog.prediction import PredictionState
+
+ if isinstance(state, str):
+ state = PredictionState(state)
+ page = self.catalog.list_predictions(
+ source=self.raw_table.identifier,
+ name=name,
+ state=state,
+ max_results=max_results,
+ page_token=page_token,
+ )
+ return page
+
def add(self, data):
arrow_table = _to_arrow_table(data, _target_schema(self.raw_table))
write_builder = self.raw_table.new_batch_write_builder()
diff --git a/paimon-python/pypaimon/tests/rest/client_test.py b/paimon-python/pypaimon/tests/rest/client_test.py
index f064cca8c3c7..4261c9ae4cf3 100644
--- a/paimon-python/pypaimon/tests/rest/client_test.py
+++ b/paimon-python/pypaimon/tests/rest/client_test.py
@@ -133,7 +133,7 @@ def test_session_request_receives_timeout_tuple(self):
client._execute_request('GET', 'http://localhost:8080/x')
self.assertTrue(req.called, "session.request must be invoked")
- kwargs = req.call_args.kwargs
+ kwargs = req.call_args[1]
self.assertEqual(
kwargs.get('timeout'), client._timeout,
"request must carry the timeout; Session.timeout is dead code")
diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py
new file mode 100644
index 000000000000..c85c03fb3b63
--- /dev/null
+++ b/paimon-python/pypaimon/tests/rest/prediction_test.py
@@ -0,0 +1,536 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+import threading
+import unittest
+from datetime import timedelta
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from unittest.mock import Mock, patch
+
+from pypaimon.api.api_response import ErrorResponse, PagedList
+from pypaimon.api.rest_exception import NoSuchResourceException
+from pypaimon.api.prediction import (
+ GetPredictionResponse,
+ ListPredictionsResponse,
+ PredictionOutput as ApiPredictionOutput,
+ PredictionSource,
+ SubmitPredictionRequest,
+)
+from pypaimon.api.rest_api import RESTApi
+from pypaimon.catalog.catalog_exception import DatabaseNotExistException
+from pypaimon.catalog.prediction import (
+ Prediction,
+ PredictionOutput,
+ PredictionState,
+ PredictionSpec,
+ PredictionStatus,
+ PredictionTerminationCode,
+ PredictionTerminationDetails,
+)
+from pypaimon.catalog.rest.rest_catalog import RESTCatalog
+from pypaimon.common.identifier import Identifier
+from pypaimon.common.json_util import JSON
+from pypaimon.common.options.core_options import CoreOptions
+from pypaimon.common.options.options import Options
+from pypaimon.multimodal.prediction_wait import PredictionWait
+from pypaimon.multimodal.table import MultimodalTable
+from pypaimon.table.file_store_table import FileStoreTable
+
+
+def _spec(**changes):
+ values = dict(
+ model="quality.qwen_vl",
+ inputs={"video": "content"},
+ instructions="Classify",
+ )
+ values.update(changes)
+ return PredictionSpec(**values)
+
+
+def _prediction(status=None, **changes):
+ if status is None:
+ status = PredictionStatus(PredictionState.PENDING)
+ values = dict(
+ prediction_id="prediction-1",
+ source=Identifier.create("quality", "clips"),
+ name="daily-quality",
+ status=status,
+ snapshot_id=7,
+ model="quality.qwen_vl",
+ )
+ values.update(changes)
+ return Prediction(**values)
+
+
+def _response(status=None, **changes):
+ if status is None:
+ status = PredictionStatus(PredictionState.PENDING)
+ values = dict(
+ prediction_id="prediction-1",
+ source=PredictionSource("quality", "clips"),
+ name="daily-quality",
+ status=status,
+ snapshot_id=7,
+ model=Identifier.create("quality", "qwen_vl"),
+ )
+ values.update(changes)
+ return GetPredictionResponse(**values)
+
+
+class PredictionTest(unittest.TestCase):
+
+ def test_request_and_response_wire_format(self):
+ request = SubmitPredictionRequest.from_spec(
+ Identifier.create("quality", "clips"), _spec())
+ body = json.loads(JSON.to_json(request))
+ self.assertTrue(body["idempotencyToken"])
+ self.assertEqual(
+ body["idempotencyToken"],
+ json.loads(JSON.to_json(request))["idempotencyToken"],
+ )
+ self.assertNotIn("resultColumns", body)
+ self.assertNotIn("mergeKeyColumns", body)
+ self.assertNotIn("prompt", body)
+ self.assertEqual("Classify", body["instructions"])
+ self.assertNotIn("outputSchema", body)
+ self.assertEqual(
+ {"database": "quality", "table": "clips"}, body["source"])
+ self.assertEqual(
+ {"database": "quality", "object": "qwen_vl"}, body["model"])
+ self.assertNotIn("snapshotId", body)
+ self.assertNotIn("tagName", body)
+
+ versioned = SubmitPredictionRequest.from_spec(
+ Identifier.create("quality", "clips"),
+ _spec(options={"scan.tag-name": "training"}),
+ )
+ versioned_body = json.loads(JSON.to_json(versioned))
+ self.assertEqual(
+ {"scan.tag-name": "training"}, versioned_body["options"])
+ self.assertNotIn("tagName", versioned_body)
+
+ keyed = SubmitPredictionRequest.from_spec(
+ Identifier.create("quality", "clips"),
+ _spec(
+ merge_key_columns=["clip_id"],
+ idempotency_token="retry-1",
+ ),
+ )
+ keyed_body = json.loads(JSON.to_json(keyed))
+ self.assertEqual(["clip_id"], keyed_body["mergeKeyColumns"])
+ self.assertEqual("retry-1", keyed_body["idempotencyToken"])
+
+ response = JSON.from_json(
+ JSON.to_json(_response()), GetPredictionResponse)
+ self.assertEqual(
+ PredictionState.PENDING, response.to_prediction().status.state)
+ self.assertIsNone(
+ response.to_prediction().status.termination_details)
+ pending_body = json.loads(JSON.to_json(_response()))
+ for field in (
+ "createdAt", "startedAt", "finishedAt",
+ "modelVersion", "outputTable", "output", "startTime",
+ "endTime"):
+ self.assertNotIn(field, pending_body)
+ terminated = PredictionStatus(
+ PredictionState.TERMINATED,
+ PredictionTerminationDetails(
+ PredictionTerminationCode.FAILED,
+ "Model request failed",
+ ),
+ )
+ completed_response = _response(
+ terminated,
+ output=ApiPredictionOutput(
+ PredictionSource("system", "prediction_results")),
+ start_time=10,
+ end_time=20,
+ )
+ terminated_body = json.loads(JSON.to_json(completed_response))
+ self.assertEqual(
+ {
+ "state": "TERMINATED",
+ "terminationDetails": {
+ "code": "FAILED",
+ "message": "Model request failed",
+ },
+ },
+ terminated_body["status"],
+ )
+ self.assertEqual(10, terminated_body["startTime"])
+ self.assertEqual(20, terminated_body["endTime"])
+ self.assertEqual(
+ {"table": {"database": "system", "table": "prediction_results"}},
+ terminated_body["output"],
+ )
+ for field in (
+ "createdAt", "startedAt", "finishedAt", "modelVersion",
+ "outputTable"):
+ self.assertNotIn(field, terminated_body)
+ completed = JSON.from_json(
+ JSON.to_json(completed_response), GetPredictionResponse)
+ self.assertEqual(
+ terminated,
+ completed.status,
+ )
+ self.assertEqual(
+ PredictionOutput(
+ Identifier.create("system", "prediction_results")),
+ completed.to_prediction().output,
+ )
+ with self.assertRaises(ValueError):
+ ApiPredictionOutput.from_dict(
+ {"artifactUri": "oss://bucket/result"})
+ page = JSON.from_json(
+ JSON.to_json(ListPredictionsResponse([_response()], "next")),
+ ListPredictionsResponse,
+ )
+ self.assertEqual("prediction-1", page.predictions[0].prediction_id)
+ self.assertEqual("next", page.next_page_token)
+
+ def test_validation(self):
+ invalid = [
+ dict(model=""),
+ dict(inputs={}),
+ dict(inputs={"video": ""}),
+ dict(merge_key_columns=["id", "id"]),
+ dict(merge_key_columns="id"),
+ dict(instructions=" "),
+ dict(options={"scan.snapshot-id": "0"}),
+ dict(options={
+ "scan.snapshot-id": "1",
+ "scan.tag-name": "v1",
+ }),
+ dict(options={"parallelism": 2}),
+ dict(idempotency_token="x" * 65),
+ ]
+ for changes in invalid:
+ with self.subTest(changes=changes), self.assertRaises(ValueError):
+ _spec(**changes)
+ with self.assertRaises(ValueError):
+ _spec(model=Identifier.create("", "qwen_vl"))
+ with self.assertRaises(ValueError):
+ PredictionStatus(PredictionState.TERMINATED)
+ with self.assertRaises(ValueError):
+ PredictionStatus(
+ PredictionState.RUNNING,
+ PredictionTerminationDetails(
+ PredictionTerminationCode.FAILED),
+ )
+
+ def test_terminating_does_not_require_final_result(self):
+ terminating = PredictionStatus(PredictionState.TERMINATING)
+ self.assertFalse(terminating.terminal)
+ for code in (
+ PredictionTerminationCode.CANCELED,
+ PredictionTerminationCode.SUCCESS,
+ ):
+ with self.subTest(code=code):
+ terminated = PredictionStatus(
+ PredictionState.TERMINATED,
+ PredictionTerminationDetails(code),
+ )
+ self.assertTrue(terminated.terminal)
+ self.assertEqual(code, terminated.termination_details.code)
+
+ def test_structured_model_preserves_dots(self):
+ request = SubmitPredictionRequest.from_spec(
+ Identifier.create("quality", "clips"),
+ _spec(model=Identifier.create("quality.archive", "qwen.vl")),
+ )
+ body = json.loads(JSON.to_json(request))
+
+ self.assertEqual(
+ {"database": "quality.archive", "object": "qwen.vl"},
+ body["model"],
+ )
+
+ def test_table_submit_fixes_view_and_returns_waiter(self):
+ raw = Mock(spec=FileStoreTable)
+ raw.options = CoreOptions(Options({"scan.snapshot-id": "7"}))
+ raw.identifier = Identifier.create("quality", "clips")
+ catalog = Mock()
+ catalog.submit_prediction.return_value = _prediction()
+ table = MultimodalTable(catalog, "quality.clips", raw)
+
+ waiter = table.predict(
+ name="daily-quality",
+ model="quality.qwen_vl",
+ inputs={"video": "content"},
+ instructions="Classify",
+ idempotency_token="retry-1",
+ )
+
+ self.assertIsInstance(waiter, PredictionWait)
+ self.assertEqual("prediction-1", waiter.prediction_id)
+ sent = catalog.submit_prediction.call_args[0][1]
+ self.assertEqual({"scan.snapshot-id": "7"}, sent.options)
+ self.assertEqual([], sent.merge_key_columns)
+ self.assertEqual("retry-1", sent.idempotency_token)
+ raw.new_read_builder.assert_not_called()
+
+ def test_table_submit_rejects_stream_scan_mode(self):
+ for table_options, request_options in (
+ ({"stream-scan-mode": "file-monitor"}, None),
+ ({}, {"stream-scan-mode": "file-monitor"}),
+ ):
+ with self.subTest(
+ table_options=table_options,
+ request_options=request_options):
+ raw = Mock(spec=FileStoreTable)
+ raw.options = CoreOptions(Options(table_options))
+ raw.identifier = Identifier.create("quality", "clips")
+ catalog = Mock()
+ table = MultimodalTable(catalog, "quality.clips", raw)
+
+ with self.assertRaisesRegex(ValueError, "stream-scan-mode"):
+ table.predict(
+ model="quality.qwen_vl",
+ inputs={"video": "content"},
+ instructions="Classify",
+ options=request_options,
+ )
+ catalog.submit_prediction.assert_not_called()
+
+ def test_table_queries_return_metadata_and_cancel_returns_waiter(self):
+ raw = Mock(spec=FileStoreTable)
+ raw.identifier = Identifier.create("quality", "clips")
+ catalog = Mock()
+ prediction = _prediction()
+ page = PagedList([prediction], "next")
+ catalog.get_prediction.return_value = prediction
+ catalog.list_predictions.return_value = page
+ catalog.cancel_prediction.return_value = prediction
+ table = MultimodalTable(catalog, "quality.clips", raw)
+
+ self.assertIs(prediction, table.get_prediction("prediction-1"))
+ self.assertIs(
+ page,
+ table.list_predictions(state=PredictionState.PENDING),
+ )
+ waiter = table.cancel_prediction("prediction-1")
+ self.assertIsInstance(waiter, PredictionWait)
+ self.assertEqual("prediction-1", waiter.prediction_id)
+
+ @patch("pypaimon.multimodal.prediction_wait.random.random", return_value=0)
+ @patch("pypaimon.multimodal.prediction_wait.time.sleep")
+ def test_result_is_client_side_polling(self, sleep, random):
+ catalog = Mock()
+ catalog.get_prediction.side_effect = [
+ _prediction(PredictionStatus(PredictionState.RUNNING)),
+ _prediction(PredictionStatus(
+ PredictionState.TERMINATED,
+ PredictionTerminationDetails(
+ PredictionTerminationCode.SUCCESS),
+ )),
+ ]
+ waiter = PredictionWait(catalog, _prediction())
+ observed = []
+ result = waiter.result(
+ timeout=timedelta(seconds=10),
+ callback=lambda prediction: observed.append(
+ prediction.status.state),
+ )
+ self.assertEqual(PredictionState.TERMINATED, result.status.state)
+ self.assertEqual(
+ PredictionTerminationCode.SUCCESS,
+ result.status.termination_details.code,
+ )
+ self.assertEqual([PredictionState.RUNNING], observed)
+ self.assertEqual(2, catalog.get_prediction.call_count)
+ sleep.assert_called_once_with(1)
+
+ def test_result_returns_failed_and_canceled_predictions(self):
+ for code in (
+ PredictionTerminationCode.FAILED,
+ PredictionTerminationCode.CANCELED,
+ ):
+ with self.subTest(code=code):
+ catalog = Mock()
+ catalog.get_prediction.return_value = _prediction(
+ PredictionStatus(
+ PredictionState.TERMINATED,
+ PredictionTerminationDetails(code, "terminal"),
+ ))
+ waiter = PredictionWait(catalog, _prediction())
+ result = waiter.result(timeout=timedelta(seconds=1))
+ self.assertEqual(
+ code, result.status.termination_details.code)
+
+ @patch(
+ "pypaimon.multimodal.prediction_wait.time.monotonic",
+ side_effect=[0, 2],
+ )
+ def test_result_timeout_does_not_cancel_prediction(self, monotonic):
+ catalog = Mock()
+ waiter = PredictionWait(catalog, _prediction())
+
+ with self.assertRaisesRegex(
+ TimeoutError, "Prediction prediction-1 did not finish"):
+ waiter.result(timeout=timedelta(seconds=1))
+
+ catalog.get_prediction.assert_not_called()
+ catalog.cancel_prediction.assert_not_called()
+
+ @patch(
+ "pypaimon.multimodal.prediction_wait.time.monotonic",
+ side_effect=[0, 0, 0.2],
+ )
+ def test_result_rejects_terminal_response_after_deadline(self, monotonic):
+ catalog = Mock()
+ catalog.get_prediction.return_value = _prediction(
+ PredictionStatus(
+ PredictionState.TERMINATED,
+ PredictionTerminationDetails(
+ PredictionTerminationCode.SUCCESS),
+ ))
+ waiter = PredictionWait(catalog, _prediction())
+
+ with self.assertRaisesRegex(
+ TimeoutError, "Prediction prediction-1 did not finish"):
+ waiter.result(timeout=timedelta(milliseconds=10))
+
+ catalog.get_prediction.assert_called_once_with("prediction-1")
+
+ def test_wait_missing_attribute_uses_attribute_error(self):
+ waiter = PredictionWait(Mock(), _prediction())
+ with self.assertRaises(AttributeError):
+ waiter.unknown_attribute
+
+ def test_untyped_prediction_404_is_not_mapped_to_missing_resource(self):
+ catalog = object.__new__(RESTCatalog)
+ catalog.rest_api = Mock()
+ untyped = NoSuchResourceException(None, None, "route not found")
+
+ catalog.rest_api.submit_prediction.side_effect = untyped
+ with self.assertRaises(NoSuchResourceException):
+ catalog.submit_prediction(
+ Identifier.create("quality", "clips"), _spec())
+
+ catalog.rest_api.get_prediction.side_effect = untyped
+ with self.assertRaises(NoSuchResourceException):
+ catalog.get_prediction("prediction-1")
+
+ catalog.rest_api.cancel_prediction.side_effect = untyped
+ with self.assertRaises(NoSuchResourceException):
+ catalog.cancel_prediction("prediction-1")
+
+ def test_submit_prediction_maps_database_404(self):
+ catalog = object.__new__(RESTCatalog)
+ catalog.rest_api = Mock()
+ catalog.rest_api.submit_prediction.side_effect = NoSuchResourceException(
+ ErrorResponse.RESOURCE_TYPE_DATABASE,
+ "models",
+ "database not found",
+ )
+
+ with self.assertRaises(DatabaseNotExistException) as context:
+ catalog.submit_prediction(
+ Identifier.create("quality", "clips"), _spec())
+
+ self.assertEqual("models", context.exception.database)
+
+ def test_rest_round_trip_paths_and_filters(self):
+ calls = []
+
+ class Handler(BaseHTTPRequestHandler):
+ def log_message(self, *args):
+ pass
+
+ def _send(self, payload, status=200):
+ body = JSON.to_json(payload).encode()
+ self.send_response(status)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+ def do_POST(self):
+ length = int(self.headers.get("Content-Length", "0"))
+ body = json.loads(self.rfile.read(length)) if length else None
+ calls.append(("POST", self.path, body))
+ status = 202 if self.path.endswith("/predictions") else 200
+ self._send(_response(), status)
+
+ def do_GET(self):
+ calls.append(("GET", self.path, None))
+ if "/predictions?" in self.path:
+ self._send(ListPredictionsResponse([_response()], "next"))
+ else:
+ self._send(_response())
+
+ server = HTTPServer(("127.0.0.1", 0), Handler)
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ try:
+ api = RESTApi(
+ {
+ "uri": "http://127.0.0.1:%s" % server.server_port,
+ "token.provider": "bear",
+ "token": "test",
+ "prefix": "catalog",
+ },
+ False,
+ )
+ catalog = object.__new__(RESTCatalog)
+ catalog.rest_api = api
+ identifier = Identifier.create("a/b", "clip table")
+
+ self.assertEqual("prediction-1", catalog.submit_prediction(
+ identifier, _spec()).prediction_id)
+ self.assertEqual(
+ "/v1/catalog/predictions",
+ calls[-1][1],
+ )
+ self.assertEqual(
+ {"database": "a/b", "table": "clip table"},
+ calls[-1][2]["source"],
+ )
+ self.assertTrue(calls[-1][2]["idempotencyToken"])
+ self.assertNotIn("mergeKeyColumns", calls[-1][2])
+
+ page = catalog.list_predictions(
+ identifier,
+ name="daily-quality",
+ state=PredictionState.RUNNING,
+ max_results=10,
+ page_token="p1",
+ )
+ self.assertEqual("prediction-1", page.elements[0].prediction_id)
+ self.assertIn("name=daily-quality", calls[-1][1])
+ self.assertIn("database=a%2Fb", calls[-1][1])
+ self.assertIn("table=clip+table", calls[-1][1])
+ self.assertIn("state=RUNNING", calls[-1][1])
+ self.assertIn("maxResults=10", calls[-1][1])
+ self.assertIn("pageToken=p1", calls[-1][1])
+
+ catalog.get_prediction("prediction/1")
+ self.assertTrue(
+ calls[-1][1].endswith("/predictions/prediction%2F1"))
+ catalog.cancel_prediction("prediction/1")
+ self.assertTrue(
+ calls[-1][1].endswith(
+ "/predictions/prediction%2F1/cancel"))
+ finally:
+ server.shutdown()
+ server.server_close()
+ thread.join()
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/paimon-python/pypaimon/tests/rest/resource_paths_test.py b/paimon-python/pypaimon/tests/rest/resource_paths_test.py
index eccd72f68f86..731e90901b89 100644
--- a/paimon-python/pypaimon/tests/rest/resource_paths_test.py
+++ b/paimon-python/pypaimon/tests/rest/resource_paths_test.py
@@ -51,6 +51,15 @@ def test_database_and_table_names_are_url_encoded(self):
"/v1/catalog%2Fid/databases/sales+db/table-details", paths.table_details("sales db"))
self.assertEqual("/v1/catalog%2Fid/tables/id/a%2Fb+c", paths.table_by_id("a/b c"))
+ def test_prediction_rejects_dot_segments(self):
+ paths = ResourcePaths("paimon")
+ for prediction_id in (".", ".."):
+ with self.subTest(prediction_id=prediction_id):
+ with self.assertRaises(ValueError):
+ paths.prediction(prediction_id)
+ with self.assertRaises(ValueError):
+ paths.cancel_prediction(prediction_id)
+
def test_signature_matches_the_path_sent(self):
received = []
diff --git a/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py b/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py
index 72cbc13e3bf4..973f9e86fd31 100644
--- a/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py
+++ b/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py
@@ -27,6 +27,7 @@
from pypaimon.api.rest_api import RESTApi
from pypaimon.api.model import CreateModelRequest
+from pypaimon.api.prediction import PredictionSource, SubmitPredictionRequest
from pypaimon.catalog.model import ModelField, ProviderModelDefinition
from pypaimon.common.identifier import Identifier
from pypaimon.function.function import FunctionImpl
@@ -44,8 +45,13 @@
from pypaimon.snapshot.snapshot_commit import PartitionStatistics
from pypaimon.table.instant import Instant
-# RESTApi methods with no entry in dlf_openapi_actions; they are signed without an action.
-WITHOUT_ACTION = frozenset()
+# REST-only APIs with no DLF POP action; they are signed without x-acs-action.
+WITHOUT_ACTION = frozenset({
+ "cancel_prediction",
+ "get_prediction",
+ "list_predictions",
+ "submit_prediction",
+})
_SPEC_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), *[os.pardir] * 4, "docs", "static")
_SPEC_FILES = ("rest-catalog-open-api.yaml", "rest-management-open-api.yaml")
@@ -63,6 +69,12 @@
[ModelField("video", "BLOB", "video")],
),
),
+ SubmitPredictionRequest: SubmitPredictionRequest(
+ source=PredictionSource("db", "t"),
+ model=Identifier.create("db", "qwen_vl"),
+ inputs={"video": "content"},
+ instructions="classify",
+ ),
str: "SELECT",
bool: False,
Identifier: Identifier.create("db", "t"),