From ba1769d9ea0ee31a2b7f3fc949bd39acfd66df9f Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Mon, 16 Nov 2026 16:31:49 +0800 Subject: [PATCH 01/41] [core][python] Add asynchronous table prediction APIs Add a provider-neutral asynchronous table prediction contract for Java and PyPaimon REST catalogs. --- docs/static/rest-catalog-open-api.yaml | 214 +++++++++++++++- .../org/apache/paimon/catalog/Prediction.java | 159 ++++++++++++ .../apache/paimon/catalog/PredictionSpec.java | 161 ++++++++++++ .../paimon/catalog/PredictionStatus.java | 36 +++ .../java/org/apache/paimon/rest/RESTApi.java | 58 +++++ .../org/apache/paimon/rest/ResourcePaths.java | 14 + .../paimon/rest/auth/DLFOpenApiActions.java | 16 ++ .../requests/SubmitPredictionRequest.java | 168 ++++++++++++ .../paimon/rest/responses/ErrorResponse.java | 1 + .../rest/responses/GetPredictionResponse.java | 203 +++++++++++++++ .../responses/ListPredictionsResponse.java | 64 +++++ .../paimon/rest/RESTApiPredictionTest.java | 143 +++++++++++ .../rest/auth/DLFOpenApiActionsTest.java | 2 +- .../RequestJacksonCompatibilityTest.java | 3 +- .../org/apache/paimon/catalog/Catalog.java | 59 +++++ .../paimon/catalog/DelegateCatalog.java | 29 +++ .../org/apache/paimon/rest/RESTCatalog.java | 75 ++++++ .../rest/RESTCatalogPredictionTest.java | 158 ++++++++++++ paimon-python/pypaimon/api/api_response.py | 1 + .../pypaimon/api/auth/dlf_openapi_actions.py | 4 + paimon-python/pypaimon/api/prediction.py | 182 +++++++++++++ paimon-python/pypaimon/api/resource_paths.py | 16 ++ paimon-python/pypaimon/api/rest_api.py | 63 +++++ paimon-python/pypaimon/catalog/catalog.py | 39 +++ .../pypaimon/catalog/catalog_exception.py | 10 + paimon-python/pypaimon/catalog/prediction.py | 146 +++++++++++ .../pypaimon/catalog/rest/rest_catalog.py | 81 ++++++ .../pypaimon/multimodal/prediction.py | 81 ++++++ paimon-python/pypaimon/multimodal/table.py | 84 ++++++ .../pypaimon/tests/rest/prediction_test.py | 242 ++++++++++++++++++ .../tests/rest/rest_api_coverage_test.py | 8 + 31 files changed, 2517 insertions(+), 3 deletions(-) create mode 100644 paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/catalog/PredictionStatus.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java create mode 100644 paimon-api/src/main/java/org/apache/paimon/rest/responses/ListPredictionsResponse.java create mode 100644 paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java create mode 100644 paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java create mode 100644 paimon-python/pypaimon/api/prediction.py create mode 100644 paimon-python/pypaimon/catalog/prediction.py create mode 100644 paimon-python/pypaimon/multimodal/prediction.py create mode 100644 paimon-python/pypaimon/tests/rest/prediction_test.py diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 40d97b3c1c7f..1fd26e7f3737 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -133,6 +133,107 @@ paths: "404": $ref: '#/components/responses/DatabaseOrModelNotExistErrorResponse' "501": {description: Models are not supported} + /v1/{prefix}/databases/{database}/tables/{table}/predictions: + post: + tags: [prediction] + summary: Submit an asynchronous table prediction + operationId: submitPrediction + description: >- + Submits an asynchronous prediction with a fixed snapshot and model version. + parameters: + - {name: prefix, in: path, required: true, schema: {type: string}} + - {name: database, in: path, required: true, schema: {type: string}} + - {name: table, in: path, required: true, schema: {type: string}} + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/SubmitPredictionRequest' + responses: + "202": + description: Prediction accepted + content: + application/json: + schema: + $ref: '#/components/schemas/GetPredictionResponse' + "400": + $ref: '#/components/responses/BadRequestErrorResponse' + "404": + $ref: '#/components/responses/TableOrModelNotExistErrorResponse' + "409": + $ref: '#/components/responses/ResourceAlreadyExistErrorResponse' + "501": {description: Predictions are not supported} + get: + tags: [prediction] + summary: List table predictions + operationId: listPredictions + description: Newest first. + parameters: + - {name: prefix, in: path, required: true, schema: {type: string}} + - {name: database, in: path, required: true, schema: {type: string}} + - {name: table, in: path, required: true, schema: {type: string}} + - {name: name, in: query, required: false, schema: {type: string}} + - name: status + in: query + required: false + schema: + $ref: '#/components/schemas/PredictionStatus' + - {name: maxResults, in: query, required: false, schema: {type: integer, format: int32, minimum: 1}} + - {name: pageToken, in: query, required: false, schema: {type: string}} + responses: + "200": + description: Prediction metadata for the table + content: + application/json: + schema: + $ref: '#/components/schemas/ListPredictionsResponse' + "404": + $ref: '#/components/responses/TableNotExistErrorResponse' + "501": {description: Predictions are not supported} + /v1/{prefix}/databases/{database}/tables/{table}/predictions/{jobId}: + get: + tags: [prediction] + summary: Get a table prediction + operationId: getPrediction + parameters: + - {name: prefix, in: path, required: true, schema: {type: string}} + - {name: database, in: path, required: true, schema: {type: string}} + - {name: table, in: path, required: true, schema: {type: string}} + - {name: jobId, in: path, required: true, schema: {type: string, minLength: 1}} + responses: + "200": + description: Prediction metadata + content: + application/json: + schema: + $ref: '#/components/schemas/GetPredictionResponse' + "404": + $ref: '#/components/responses/TableOrPredictionNotExistErrorResponse' + "501": {description: Predictions are not supported} + /v1/{prefix}/databases/{database}/tables/{table}/predictions/{jobId}/cancel: + post: + tags: [prediction] + summary: Cancel a table prediction + operationId: cancelPrediction + description: >- + Requests cancellation and returns the latest state. Terminal jobs are + unchanged. + parameters: + - {name: prefix, in: path, required: true, schema: {type: string}} + - {name: database, in: path, required: true, schema: {type: string}} + - {name: table, in: path, required: true, schema: {type: string}} + - {name: jobId, in: path, required: true, schema: {type: string, minLength: 1}} + responses: + "200": + description: Latest prediction metadata + content: + application/json: + schema: + $ref: '#/components/schemas/GetPredictionResponse' + "404": + $ref: '#/components/responses/TableOrPredictionNotExistErrorResponse' + "501": {description: Predictions are not supported} /v1/config: get: tags: @@ -2881,6 +2982,32 @@ components: resourceType: MODEL resourceName: model code: 404 + TableOrModelNotExistErrorResponse: + description: Database, table, or model not found + content: + application/json: + schema: + allOf: + - $ref: '#/components/schemas/ErrorResponse' + - type: object + required: [ resourceType ] + properties: + resourceType: + type: string + enum: [ "DATABASE", "TABLE", "MODEL" ] + TableOrPredictionNotExistErrorResponse: + description: Database, table, or prediction not found + content: + application/json: + schema: + allOf: + - $ref: '#/components/schemas/ErrorResponse' + - type: object + required: [ resourceType ] + properties: + resourceType: + type: string + enum: [ "DATABASE", "TABLE", "PREDICTION" ] TableNotExistErrorResponse: description: Not Found - TableNotExistException, the table does not exist @@ -3264,7 +3391,7 @@ components: type: string resourceType: type: [ string, "null" ] - enum: [ "DATABASE", "TABLE", "PARTITION", "COLUMN", "SNAPSHOT", "BRANCH", "TAG", "VIEW", "SEMANTIC_VIEW", "DIALECT", "FUNCTION", "MODEL", "DEFINITION", "UNKNOWN", null ] + enum: [ "DATABASE", "TABLE", "PARTITION", "COLUMN", "SNAPSHOT", "BRANCH", "TAG", "VIEW", "SEMANTIC_VIEW", "DIALECT", "FUNCTION", "MODEL", "PREDICTION", "DEFINITION", "UNKNOWN", null ] resourceName: type: [ string, "null" ] code: @@ -4548,6 +4675,91 @@ components: $ref: '#/components/schemas/GetModelResponse' nextPageToken: type: [ string, "null" ] + PredictionStatus: + type: string + enum: [PENDING, RUNNING, SUCCEEDED, PARTIAL_SUCCESS, FAILED, CANCELLED] + SubmitPredictionRequest: + type: object + additionalProperties: false + required: [requestId, model, inputs, resultColumns, prompt, outputSchema] + properties: + name: + type: string + minLength: 1 + description: Optional non-unique label. + requestId: + type: string + minLength: 1 + description: Client-generated idempotency key. + model: + type: string + minLength: 1 + description: Database-qualified model name. + inputs: + type: object + minProperties: 1 + additionalProperties: + type: string + minLength: 1 + description: Model input-to-column mappings. + resultColumns: + type: array + uniqueItems: true + items: {type: string, minLength: 1} + prompt: + type: string + minLength: 1 + outputSchema: + type: object + minProperties: 1 + additionalProperties: + type: string + minLength: 1 + description: Paimon output column types. + where: + type: string + minLength: 1 + snapshotId: + type: integer + format: int64 + minimum: 1 + tagName: + type: string + minLength: 1 + options: + type: object + additionalProperties: true + description: snapshotId and tagName are mutually exclusive. + GetPredictionResponse: + type: object + additionalProperties: false + required: [jobId, status, snapshotId, model, modelVersion, outputTable, createdAt] + properties: + jobId: {type: string, minLength: 1} + name: {type: string, minLength: 1} + status: + $ref: '#/components/schemas/PredictionStatus' + snapshotId: {type: integer, format: int64, minimum: 1} + model: {type: string, minLength: 1} + modelVersion: {type: string, minLength: 1} + outputTable: {type: string, minLength: 1} + createdAt: {type: integer, format: int64, minimum: 0} + startedAt: {type: integer, format: int64, minimum: 0} + finishedAt: {type: integer, format: int64, minimum: 0} + error: {type: string} + total: {type: integer, format: int64, minimum: 0} + succeeded: {type: integer, format: int64, minimum: 0} + failed: {type: integer, format: int64, minimum: 0} + ListPredictionsResponse: + type: object + required: [predictions] + properties: + predictions: + type: array + items: + $ref: '#/components/schemas/GetPredictionResponse' + nextPageToken: + type: [ string, "null" ] ConsumerInfo: type: object properties: diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java new file mode 100644 index 000000000000..4ec297abf071 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.catalog; + +import org.apache.paimon.annotation.Experimental; + +import javax.annotation.Nullable; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Durable metadata for one asynchronous table prediction. */ +@Experimental +public class Prediction { + + private final String jobId; + private final @Nullable String name; + private final PredictionStatus status; + private final long snapshotId; + private final String model; + private final String modelVersion; + private final String outputTable; + private final long createdAt; + private final @Nullable Long startedAt; + private final @Nullable Long finishedAt; + private final @Nullable String error; + private final @Nullable Long total; + private final @Nullable Long succeeded; + private final @Nullable Long failed; + + public Prediction( + String jobId, + @Nullable String name, + PredictionStatus status, + long snapshotId, + String model, + String modelVersion, + String outputTable, + long createdAt, + @Nullable Long startedAt, + @Nullable Long finishedAt, + @Nullable String error, + @Nullable Long total, + @Nullable Long succeeded, + @Nullable Long failed) { + checkNonBlank(jobId, "jobId"); + checkArgument(status != null, "status must not be null"); + checkArgument(snapshotId > 0, "snapshotId must be positive"); + checkNonBlank(model, "model"); + checkNonBlank(modelVersion, "modelVersion"); + checkNonBlank(outputTable, "outputTable"); + checkArgument(createdAt >= 0, "createdAt must not be negative"); + checkOptionalNonNegative(startedAt, "startedAt"); + checkOptionalNonNegative(finishedAt, "finishedAt"); + checkOptionalNonNegative(total, "total"); + checkOptionalNonNegative(succeeded, "succeeded"); + checkOptionalNonNegative(failed, "failed"); + this.jobId = jobId; + this.name = name; + this.status = status; + this.snapshotId = snapshotId; + this.model = model; + this.modelVersion = modelVersion; + this.outputTable = outputTable; + this.createdAt = createdAt; + this.startedAt = startedAt; + this.finishedAt = finishedAt; + this.error = error; + this.total = total; + this.succeeded = succeeded; + this.failed = failed; + } + + public String jobId() { + return jobId; + } + + @Nullable + public String name() { + return name; + } + + public PredictionStatus status() { + return status; + } + + public long snapshotId() { + return snapshotId; + } + + public String model() { + return model; + } + + public String modelVersion() { + return modelVersion; + } + + public String outputTable() { + return outputTable; + } + + public long createdAt() { + return createdAt; + } + + @Nullable + public Long startedAt() { + return startedAt; + } + + @Nullable + public Long finishedAt() { + return finishedAt; + } + + @Nullable + public String error() { + return error; + } + + @Nullable + public Long total() { + return total; + } + + @Nullable + public Long succeeded() { + return succeeded; + } + + @Nullable + public Long failed() { + return failed; + } + + private static void checkNonBlank(String value, String name) { + checkArgument(value != null && !value.trim().isEmpty(), "%s must not be blank", name); + } + + private static void checkOptionalNonNegative(@Nullable Long value, String name) { + checkArgument(value == null || value >= 0, "%s must not be negative", name); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java new file mode 100644 index 000000000000..91252778dfb7 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.catalog; + +import org.apache.paimon.annotation.Experimental; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Provider-neutral specification for an asynchronous prediction over a table snapshot. */ +@Experimental +public class PredictionSpec { + + private final @Nullable String name; + private final String model; + private final Map inputs; + private final List resultColumns; + private final String prompt; + private final Map outputSchema; + private final @Nullable String where; + private final @Nullable Long snapshotId; + private final @Nullable String tagName; + private final Map options; + + public PredictionSpec( + @Nullable String name, + String model, + Map inputs, + List resultColumns, + String prompt, + Map outputSchema, + @Nullable String where, + @Nullable Long snapshotId, + @Nullable String tagName, + @Nullable Map options) { + checkOptionalNonBlank(name, "name"); + checkNonBlank(model, "model"); + checkNonBlank(prompt, "prompt"); + checkOptionalNonBlank(where, "where"); + checkOptionalNonBlank(tagName, "tagName"); + checkArgument(snapshotId == null || snapshotId > 0, "snapshotId must be positive"); + checkArgument( + snapshotId == null || tagName == null, + "snapshotId and tagName are mutually exclusive"); + this.name = name; + this.model = model; + this.inputs = copyStringMap(inputs, "inputs", true); + this.resultColumns = copyStringList(resultColumns, "resultColumns"); + this.prompt = prompt; + this.outputSchema = copyStringMap(outputSchema, "outputSchema", true); + this.where = where; + this.snapshotId = snapshotId; + this.tagName = tagName; + this.options = + options == null + ? Collections.emptyMap() + : Collections.unmodifiableMap(new LinkedHashMap<>(options)); + } + + @Nullable + public String name() { + return name; + } + + public String model() { + return model; + } + + public Map inputs() { + return inputs; + } + + public List resultColumns() { + return resultColumns; + } + + public String prompt() { + return prompt; + } + + public Map outputSchema() { + return outputSchema; + } + + @Nullable + public String where() { + return where; + } + + @Nullable + public Long snapshotId() { + return snapshotId; + } + + @Nullable + public String tagName() { + return tagName; + } + + public Map options() { + return options; + } + + private static Map copyStringMap( + Map values, String name, boolean nonEmpty) { + checkArgument(values != null, "%s must not be null", name); + checkArgument(!nonEmpty || !values.isEmpty(), "%s must not be empty", name); + Map copy = new LinkedHashMap<>(); + values.forEach( + (key, value) -> { + checkNonBlank(key, name + " key"); + checkNonBlank(value, name + " value"); + copy.put(key, value); + }); + return Collections.unmodifiableMap(copy); + } + + private static List copyStringList(List values, String name) { + checkArgument(values != null, "%s must not be null", name); + List copy = new ArrayList<>(); + for (String value : values) { + checkNonBlank(value, name + " value"); + checkArgument(!copy.contains(value), "%s must not contain duplicates", name); + copy.add(value); + } + return Collections.unmodifiableList(copy); + } + + private static void checkOptionalNonBlank(@Nullable String value, String name) { + if (value != null) { + checkNonBlank(value, name); + } + } + + private static void checkNonBlank(String value, String name) { + checkArgument(value != null && !value.trim().isEmpty(), "%s must not be blank", name); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionStatus.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionStatus.java new file mode 100644 index 000000000000..5f915d61c258 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionStatus.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.catalog; + +import org.apache.paimon.annotation.Experimental; + +/** Lifecycle state of an asynchronous table prediction. */ +@Experimental +public enum PredictionStatus { + PENDING, + RUNNING, + SUCCEEDED, + PARTIAL_SUCCESS, + FAILED, + CANCELLED; + + public boolean isTerminal() { + return this == SUCCEEDED || this == PARTIAL_SUCCESS || this == FAILED || this == CANCELLED; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index d017d1544848..0297ff536f97 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -24,6 +24,7 @@ import org.apache.paimon.annotation.Public; import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.PredictionStatus; import org.apache.paimon.consumer.ConsumerInfo; import org.apache.paimon.function.FunctionChange; import org.apache.paimon.management.DataPolicy; @@ -72,6 +73,7 @@ import org.apache.paimon.rest.requests.RevokePermissionRequest; import org.apache.paimon.rest.requests.RollbackSchemaRequest; import org.apache.paimon.rest.requests.RollbackTableRequest; +import org.apache.paimon.rest.requests.SubmitPredictionRequest; import org.apache.paimon.rest.requests.UpsertLabelRequest; import org.apache.paimon.rest.requests.UpsertSemanticViewRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; @@ -86,6 +88,7 @@ import org.apache.paimon.rest.responses.GetFunctionResponse; import org.apache.paimon.rest.responses.GetLabelResponse; import org.apache.paimon.rest.responses.GetModelResponse; +import org.apache.paimon.rest.responses.GetPredictionResponse; import org.apache.paimon.rest.responses.GetSchemaResponse; import org.apache.paimon.rest.responses.GetSemanticViewResponse; import org.apache.paimon.rest.responses.GetTableResponse; @@ -105,6 +108,7 @@ import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListPermissionsResponse; import org.apache.paimon.rest.responses.ListPoliciesResponse; +import org.apache.paimon.rest.responses.ListPredictionsResponse; import org.apache.paimon.rest.responses.ListSchemasResponse; import org.apache.paimon.rest.responses.ListSemanticViewsResponse; import org.apache.paimon.rest.responses.ListSnapshotsResponse; @@ -204,6 +208,8 @@ public class RESTApi { public static final String TAG_NAME_PREFIX = "tagNamePrefix"; public static final String ENTITY_TYPE = "entityType"; public static final String ENTITY_NAME = "entityName"; + public static final String PREDICTION_NAME = "name"; + public static final String PREDICTION_STATUS = "status"; public static final long TOKEN_EXPIRATION_SAFE_TIME_MILLIS = 3_600_000L; @@ -1812,6 +1818,58 @@ public void dropModel(Identifier identifier) { restAuthFunction); } + /** Submit an asynchronous prediction over a table. */ + public GetPredictionResponse submitPrediction( + Identifier identifier, SubmitPredictionRequest request) { + return client.post( + resourcePaths.predictions(identifier.getDatabaseName(), identifier.getObjectName()), + request, + GetPredictionResponse.class, + restAuthFunction); + } + + /** Get one asynchronous table prediction by its unique job id. */ + public GetPredictionResponse getPrediction(Identifier identifier, String jobId) { + return client.get( + resourcePaths.prediction( + identifier.getDatabaseName(), identifier.getObjectName(), jobId), + GetPredictionResponse.class, + restAuthFunction); + } + + /** List asynchronous predictions for a table, ordered newest first by the server. */ + public PagedList listPredictions( + Identifier identifier, + @Nullable String name, + @Nullable PredictionStatus status, + @Nullable Integer maxResults, + @Nullable String pageToken) { + Map queryParams = + buildPagedQueryParams( + maxResults, + pageToken, + Pair.of(PREDICTION_NAME, name), + Pair.of(PREDICTION_STATUS, status == null ? null : status.name())); + ListPredictionsResponse response = + client.get( + resourcePaths.predictions( + identifier.getDatabaseName(), identifier.getObjectName()), + queryParams, + ListPredictionsResponse.class, + restAuthFunction); + return new PagedList<>(response.getPredictions(), response.getNextPageToken()); + } + + /** Request cancellation and return the updated prediction metadata. */ + public GetPredictionResponse cancelPrediction(Identifier identifier, String jobId) { + return client.post( + resourcePaths.cancelPrediction( + identifier.getDatabaseName(), identifier.getObjectName(), jobId), + null, + GetPredictionResponse.class, + restAuthFunction); + } + /** * Alter a function. * diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java index dfbf2b489a31..da5975bd6ddd 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -47,6 +47,7 @@ public class ResourcePaths { protected static final String REGISTER = "register"; protected static final String FUNCTIONS = "functions"; protected static final String MODELS = "models"; + protected static final String PREDICTIONS = "predictions"; protected static final String FUNCTION_DETAILS = "function-details"; protected static final String PERMISSIONS = "permissions"; protected static final String POLICIES = "policies"; @@ -160,6 +161,19 @@ public String model(String databaseName, String modelName) { return SLASH.join(models(databaseName), encodeString(modelName)); } + public String predictions(String databaseName, String tableName) { + return SLASH.join(table(databaseName, tableName), PREDICTIONS); + } + + public String prediction(String databaseName, String tableName, String jobId) { + checkArgument(jobId != null && !jobId.trim().isEmpty(), "jobId must not be blank"); + return SLASH.join(predictions(databaseName, tableName), encodeString(jobId)); + } + + public String cancelPrediction(String databaseName, String tableName, String jobId) { + return SLASH.join(prediction(databaseName, tableName, jobId), "cancel"); + } + private static void checkModelPathSegment(String value, String name) { checkArgument(value != null && !value.trim().isEmpty(), "%s must not be blank", name); checkArgument( diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java index 109dcbd5b263..3cd08cfe7f83 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java @@ -51,6 +51,22 @@ final class DLFOpenApiActions { {"GET", "/v1/{prefix}/tables/id/{tableId}", "GetTableById"}, {"POST", "/v1/{prefix}/tables/rename", "RenameTable"}, {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/auth", "AuthTableQuery"}, + { + "POST", + "/v1/{prefix}/databases/{database}/tables/{table}/predictions", + "SubmitPrediction" + }, + {"GET", "/v1/{prefix}/databases/{database}/tables/{table}/predictions", "ListPredictions"}, + { + "GET", + "/v1/{prefix}/databases/{database}/tables/{table}/predictions/{jobId}", + "GetPrediction" + }, + { + "POST", + "/v1/{prefix}/databases/{database}/tables/{table}/predictions/{jobId}/cancel", + "CancelPrediction" + }, {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/commit", "CommitTable"}, { "POST", diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java new file mode 100644 index 000000000000..88e03e680c16 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.catalog.PredictionSpec; +import org.apache.paimon.rest.RESTRequest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Request for submitting an asynchronous table prediction. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class SubmitPredictionRequest implements RESTRequest { + + private static final String FIELD_NAME = "name"; + private static final String FIELD_REQUEST_ID = "requestId"; + private static final String FIELD_MODEL = "model"; + private static final String FIELD_INPUTS = "inputs"; + private static final String FIELD_RESULT_COLUMNS = "resultColumns"; + private static final String FIELD_PROMPT = "prompt"; + private static final String FIELD_OUTPUT_SCHEMA = "outputSchema"; + private static final String FIELD_WHERE = "where"; + private static final String FIELD_SNAPSHOT_ID = "snapshotId"; + private static final String FIELD_TAG_NAME = "tagName"; + private static final String FIELD_OPTIONS = "options"; + + private final String requestId; + private final PredictionSpec spec; + + @JsonCreator + public SubmitPredictionRequest( + @Nullable @JsonProperty(FIELD_NAME) String name, + @Nullable @JsonProperty(FIELD_REQUEST_ID) String requestId, + @JsonProperty(FIELD_MODEL) String model, + @JsonProperty(FIELD_INPUTS) Map inputs, + @JsonProperty(FIELD_RESULT_COLUMNS) List resultColumns, + @JsonProperty(FIELD_PROMPT) String prompt, + @JsonProperty(FIELD_OUTPUT_SCHEMA) Map outputSchema, + @Nullable @JsonProperty(FIELD_WHERE) String where, + @Nullable @JsonProperty(FIELD_SNAPSHOT_ID) Long snapshotId, + @Nullable @JsonProperty(FIELD_TAG_NAME) String tagName, + @Nullable @JsonProperty(FIELD_OPTIONS) Map options) { + checkArgument( + requestId == null || !requestId.trim().isEmpty(), "requestId must not be blank"); + this.requestId = requestId == null ? UUID.randomUUID().toString() : requestId; + this.spec = + new PredictionSpec( + name, + model, + inputs, + resultColumns, + prompt, + outputSchema, + where, + snapshotId, + tagName, + options); + } + + public SubmitPredictionRequest(PredictionSpec spec) { + this( + spec.name(), + null, + spec.model(), + spec.inputs(), + spec.resultColumns(), + spec.prompt(), + spec.outputSchema(), + spec.where(), + spec.snapshotId(), + spec.tagName(), + spec.options()); + } + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter(FIELD_NAME) + public String getName() { + return spec.name(); + } + + @JsonGetter(FIELD_REQUEST_ID) + public String getRequestId() { + return requestId; + } + + @JsonGetter(FIELD_MODEL) + public String getModel() { + return spec.model(); + } + + @JsonGetter(FIELD_INPUTS) + public Map getInputs() { + return spec.inputs(); + } + + @JsonGetter(FIELD_RESULT_COLUMNS) + public List getResultColumns() { + return spec.resultColumns(); + } + + @JsonGetter(FIELD_PROMPT) + public String getPrompt() { + return spec.prompt(); + } + + @JsonGetter(FIELD_OUTPUT_SCHEMA) + public Map getOutputSchema() { + return spec.outputSchema(); + } + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter(FIELD_WHERE) + public String getWhere() { + return spec.where(); + } + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter(FIELD_SNAPSHOT_ID) + public Long getSnapshotId() { + return spec.snapshotId(); + } + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter(FIELD_TAG_NAME) + public String getTagName() { + return spec.tagName(); + } + + @JsonGetter(FIELD_OPTIONS) + public Map getOptions() { + return spec.options(); + } + + public PredictionSpec toPredictionSpec() { + return spec; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java index b5a6f1c792a7..ff47500d40f8 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java @@ -57,6 +57,7 @@ public class ErrorResponse implements RESTResponse { public static final String RESOURCE_TYPE_FUNCTION = "FUNCTION"; public static final String RESOURCE_TYPE_MODEL = "MODEL"; + public static final String RESOURCE_TYPE_PREDICTION = "PREDICTION"; public static final String RESOURCE_TYPE_POLICY = "POLICY"; diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java new file mode 100644 index 000000000000..c6da27abccc9 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.catalog.Prediction; +import org.apache.paimon.catalog.PredictionStatus; +import org.apache.paimon.rest.RESTResponse; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +/** Response containing durable metadata for one asynchronous table prediction. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class GetPredictionResponse implements RESTResponse { + + private final String jobId; + private final @Nullable String name; + private final PredictionStatus status; + private final long snapshotId; + private final String model; + private final String modelVersion; + private final String outputTable; + private final long createdAt; + private final @Nullable Long startedAt; + private final @Nullable Long finishedAt; + private final @Nullable String error; + private final @Nullable Long total; + private final @Nullable Long succeeded; + private final @Nullable Long failed; + + @JsonCreator + public GetPredictionResponse( + @JsonProperty("jobId") String jobId, + @Nullable @JsonProperty("name") String name, + @JsonProperty("status") PredictionStatus status, + @JsonProperty("snapshotId") long snapshotId, + @JsonProperty("model") String model, + @JsonProperty("modelVersion") String modelVersion, + @JsonProperty("outputTable") String outputTable, + @JsonProperty("createdAt") long createdAt, + @Nullable @JsonProperty("startedAt") Long startedAt, + @Nullable @JsonProperty("finishedAt") Long finishedAt, + @Nullable @JsonProperty("error") String error, + @Nullable @JsonProperty("total") Long total, + @Nullable @JsonProperty("succeeded") Long succeeded, + @Nullable @JsonProperty("failed") Long failed) { + this.jobId = jobId; + this.name = name; + this.status = status; + this.snapshotId = snapshotId; + this.model = model; + this.modelVersion = modelVersion; + this.outputTable = outputTable; + this.createdAt = createdAt; + this.startedAt = startedAt; + this.finishedAt = finishedAt; + this.error = error; + this.total = total; + this.succeeded = succeeded; + this.failed = failed; + } + + public GetPredictionResponse(Prediction prediction) { + this( + prediction.jobId(), + prediction.name(), + prediction.status(), + prediction.snapshotId(), + prediction.model(), + prediction.modelVersion(), + prediction.outputTable(), + prediction.createdAt(), + prediction.startedAt(), + prediction.finishedAt(), + prediction.error(), + prediction.total(), + prediction.succeeded(), + prediction.failed()); + } + + @JsonGetter("jobId") + public String getJobId() { + return jobId; + } + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter("name") + public String getName() { + return name; + } + + @JsonGetter("status") + public PredictionStatus getStatus() { + return status; + } + + @JsonGetter("snapshotId") + public long getSnapshotId() { + return snapshotId; + } + + @JsonGetter("model") + public String getModel() { + return model; + } + + @JsonGetter("modelVersion") + public String getModelVersion() { + return modelVersion; + } + + @JsonGetter("outputTable") + public String getOutputTable() { + return outputTable; + } + + @JsonGetter("createdAt") + public long getCreatedAt() { + return createdAt; + } + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter("startedAt") + public Long getStartedAt() { + return startedAt; + } + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter("finishedAt") + public Long getFinishedAt() { + return finishedAt; + } + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter("error") + public String getError() { + return error; + } + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter("total") + public Long getTotal() { + return total; + } + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter("succeeded") + public Long getSucceeded() { + return succeeded; + } + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter("failed") + public Long getFailed() { + return failed; + } + + public Prediction toPrediction() { + return new Prediction( + jobId, + name, + status, + snapshotId, + model, + modelVersion, + outputTable, + createdAt, + startedAt, + finishedAt, + error, + total, + succeeded, + failed); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListPredictionsResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListPredictionsResponse.java new file mode 100644 index 000000000000..a45cecfba285 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListPredictionsResponse.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import java.util.Collections; +import java.util.List; + +/** Response for listing asynchronous table predictions. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class ListPredictionsResponse implements PagedResponse { + + private final List predictions; + private final @Nullable String nextPageToken; + + @JsonCreator + public ListPredictionsResponse( + @Nullable @JsonProperty("predictions") List predictions, + @Nullable @JsonProperty("nextPageToken") String nextPageToken) { + this.predictions = predictions == null ? Collections.emptyList() : predictions; + this.nextPageToken = nextPageToken; + } + + @JsonGetter("predictions") + public List getPredictions() { + return predictions; + } + + @Override + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter("nextPageToken") + public String getNextPageToken() { + return nextPageToken; + } + + @Override + public List data() { + return predictions; + } +} diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java new file mode 100644 index 000000000000..8ceb09f1edb8 --- /dev/null +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.catalog.Prediction; +import org.apache.paimon.catalog.PredictionSpec; +import org.apache.paimon.catalog.PredictionStatus; +import org.apache.paimon.rest.requests.SubmitPredictionRequest; +import org.apache.paimon.rest.responses.GetPredictionResponse; +import org.apache.paimon.rest.responses.ListPredictionsResponse; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Wire-format tests for asynchronous table prediction metadata. */ +class RESTApiPredictionTest { + + @Test + void testRequestAndResponseWireFormat() throws Exception { + SubmitPredictionRequest request = new SubmitPredictionRequest(spec()); + Map requestMap = RESTApi.fromJson(RESTApi.toJson(request), Map.class); + assertThat(requestMap.get("requestId")).isInstanceOf(String.class); + assertThat((String) requestMap.get("requestId")).isNotBlank(); + assertThat(requestMap) + .containsEntry("model", "quality.qwen_vl") + .doesNotContainKeys("name", "where", "snapshotId", "tagName"); + assertThat(requestMap.get("resultColumns")).isEqualTo(Collections.singletonList("clip_id")); + + GetPredictionResponse response = new GetPredictionResponse(prediction()); + GetPredictionResponse decoded = + RESTApi.fromJson(RESTApi.toJson(response), GetPredictionResponse.class); + assertThat(decoded.toPrediction().status()).isEqualTo(PredictionStatus.PENDING); + assertThat(decoded.getName()).isNull(); + + ListPredictionsResponse empty = RESTApi.fromJson("{}", ListPredictionsResponse.class); + assertThat(empty.getPredictions()).isEmpty(); + assertThat(RESTApi.fromJson(RESTApi.toJson(empty), Map.class)) + .containsOnlyKeys("predictions") + .containsEntry("predictions", Collections.emptyList()); + } + + @Test + void testSpecValidation() { + assertThatThrownBy( + () -> + new PredictionSpec( + null, + "quality.qwen_vl", + Collections.emptyMap(), + Collections.emptyList(), + "Classify", + Collections.singletonMap("result", "STRING"), + null, + null, + null, + Collections.emptyMap())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("inputs must not be empty"); + + assertThatThrownBy( + () -> + new PredictionSpec( + null, + "quality.qwen_vl", + Collections.singletonMap("video", "content"), + Arrays.asList("clip_id", "clip_id"), + "Classify", + Collections.singletonMap("result", "STRING"), + null, + null, + null, + Collections.emptyMap())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("resultColumns must not contain duplicates"); + } + + @Test + void testTerminalStates() { + assertThat(PredictionStatus.PENDING.isTerminal()).isFalse(); + assertThat(PredictionStatus.RUNNING.isTerminal()).isFalse(); + assertThat(PredictionStatus.SUCCEEDED.isTerminal()).isTrue(); + assertThat(PredictionStatus.PARTIAL_SUCCESS.isTerminal()).isTrue(); + assertThat(PredictionStatus.FAILED.isTerminal()).isTrue(); + assertThat(PredictionStatus.CANCELLED.isTerminal()).isTrue(); + } + + private static PredictionSpec spec() { + Map inputs = new LinkedHashMap<>(); + inputs.put("video", "content"); + return new PredictionSpec( + null, + "quality.qwen_vl", + inputs, + Collections.singletonList("clip_id"), + "Classify", + Collections.singletonMap("result", "STRING"), + null, + null, + null, + Collections.emptyMap()); + } + + private static Prediction prediction() { + return new Prediction( + "job-1", + null, + PredictionStatus.PENDING, + 7, + "quality.qwen_vl", + "1", + "system.prediction_results", + 10, + null, + null, + null, + null, + null, + null); + } +} diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiActionsTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiActionsTest.java index 9e4a4f1c36b3..6d741fd6828a 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiActionsTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiActionsTest.java @@ -49,7 +49,7 @@ public void testEveryOperationResolvesToItself() { assertEquals(operation[2], DLFOpenApiActions.resolve(operation[0], path), path); actions.add(operation[2]); } - assertEquals(79, actions.size()); + assertEquals(83, actions.size()); } /** The table and docs/static/rest-*-open-api.yaml must list the same operations. */ diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java index 3c2c64357630..248c6d75c115 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java @@ -225,7 +225,8 @@ public class RequestJacksonCompatibilityTest { RenameTableRequest.class, ReplaceTableRequest.class, RevokePermissionRequest.class, - RollbackTableRequest.class) + RollbackTableRequest.class, + SubmitPredictionRequest.class) .collect(Collectors.toSet()); @ParameterizedTest(name = "{0}") diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java index 0aa2795bad9e..d2c684d25ff4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java @@ -1197,6 +1197,37 @@ default void dropModel(Identifier identifier, boolean ignoreIfNotExists) throw new UnsupportedOperationException("Current Catalog does not support models"); } + // ======================= Prediction methods =============================== + + /** Submit an asynchronous prediction over a table and its fixed snapshot. */ + default Prediction submitPrediction(Identifier identifier, PredictionSpec spec) + throws TableNotExistException, ModelNotExistException { + throw new UnsupportedOperationException("Current Catalog does not support predictions"); + } + + /** Get one asynchronous table prediction by its unique job id. */ + default Prediction getPrediction(Identifier identifier, String jobId) + throws TableNotExistException, PredictionNotExistException { + throw new UnsupportedOperationException("Current Catalog does not support predictions"); + } + + /** List asynchronous predictions for a table. */ + default PagedList listPredictions( + Identifier identifier, + @Nullable String name, + @Nullable PredictionStatus status, + @Nullable Integer maxResults, + @Nullable String pageToken) + throws TableNotExistException { + throw new UnsupportedOperationException("Current Catalog does not support predictions"); + } + + /** Request cancellation and return the updated prediction metadata. */ + default Prediction cancelPrediction(Identifier identifier, String jobId) + throws TableNotExistException, PredictionNotExistException { + throw new UnsupportedOperationException("Current Catalog does not support predictions"); + } + // ======================= Function methods =============================== /** @@ -1937,6 +1968,34 @@ public Identifier identifier() { } } + /** Exception for trying to access a prediction that doesn't exist. */ + class PredictionNotExistException extends Exception { + + private static final String MSG = "Prediction %s for table %s doesn't exist."; + + private final Identifier identifier; + private final String jobId; + + public PredictionNotExistException(Identifier identifier, String jobId) { + this(identifier, jobId, null); + } + + public PredictionNotExistException( + Identifier identifier, String jobId, @Nullable Throwable cause) { + super(String.format(MSG, jobId, identifier.getFullName()), cause); + this.identifier = identifier; + this.jobId = jobId; + } + + public Identifier identifier() { + return identifier; + } + + public String jobId() { + return jobId; + } + } + /** Exception for trying to create a function that already exists. */ class FunctionAlreadyExistException extends Exception { diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java index 48383b090066..08579b26fd88 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java @@ -407,6 +407,35 @@ public void dropModel(Identifier identifier, boolean ignoreIfNotExists) wrapped.dropModel(identifier, ignoreIfNotExists); } + @Override + public Prediction submitPrediction(Identifier identifier, PredictionSpec spec) + throws TableNotExistException, ModelNotExistException { + return wrapped.submitPrediction(identifier, spec); + } + + @Override + public Prediction getPrediction(Identifier identifier, String jobId) + throws TableNotExistException, PredictionNotExistException { + return wrapped.getPrediction(identifier, jobId); + } + + @Override + public PagedList listPredictions( + Identifier identifier, + @Nullable String name, + @Nullable PredictionStatus status, + @Nullable Integer maxResults, + @Nullable String pageToken) + throws TableNotExistException { + return wrapped.listPredictions(identifier, name, status, maxResults, pageToken); + } + + @Override + public Prediction cancelPrediction(Identifier identifier, String jobId) + throws TableNotExistException, PredictionNotExistException { + return wrapped.cancelPrediction(identifier, jobId); + } + @Override public List listFunctions(String databaseName) throws DatabaseNotExistException { return wrapped.listFunctions(databaseName); diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index fd5a0df13732..a19de2e013bb 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -31,6 +31,9 @@ import org.apache.paimon.catalog.Database; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.catalog.Model; +import org.apache.paimon.catalog.Prediction; +import org.apache.paimon.catalog.PredictionSpec; +import org.apache.paimon.catalog.PredictionStatus; import org.apache.paimon.catalog.PropertyChange; import org.apache.paimon.catalog.TableMetadata; import org.apache.paimon.catalog.TableQueryAuthResult; @@ -58,6 +61,7 @@ import org.apache.paimon.rest.exceptions.NotImplementedException; import org.apache.paimon.rest.exceptions.ServiceFailureException; import org.apache.paimon.rest.requests.CreateModelRequest; +import org.apache.paimon.rest.requests.SubmitPredictionRequest; import org.apache.paimon.rest.responses.AuthTableQueryResponse; import org.apache.paimon.rest.responses.ErrorResponse; import org.apache.paimon.rest.responses.GetDatabaseResponse; @@ -1210,6 +1214,77 @@ public void dropModel(Identifier identifier, boolean ignoreIfNotExists) } } + @Override + public Prediction submitPrediction(Identifier identifier, PredictionSpec spec) + throws TableNotExistException, ModelNotExistException { + try { + return api.submitPrediction(identifier, new SubmitPredictionRequest(spec)) + .toPrediction(); + } catch (NoSuchResourceException e) { + if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_MODEL)) { + throw new ModelNotExistException(Identifier.fromString(spec.model()), e); + } + throw new TableNotExistException(identifier, e); + } catch (NotImplementedException e) { + throw new UnsupportedOperationException("REST Catalog does not support predictions", e); + } + } + + @Override + public Prediction getPrediction(Identifier identifier, String jobId) + throws TableNotExistException, PredictionNotExistException { + try { + return api.getPrediction(identifier, jobId).toPrediction(); + } catch (NoSuchResourceException e) { + if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_PREDICTION)) { + throw new PredictionNotExistException(identifier, jobId, e); + } + throw new TableNotExistException(identifier, e); + } catch (NotImplementedException e) { + throw new UnsupportedOperationException("REST Catalog does not support predictions", e); + } + } + + @Override + public PagedList listPredictions( + Identifier identifier, + @Nullable String name, + @Nullable PredictionStatus status, + @Nullable Integer maxResults, + @Nullable String pageToken) + throws TableNotExistException { + try { + PagedList response = + api.listPredictions(identifier, name, status, maxResults, pageToken); + return new PagedList<>( + response.getElements().stream() + .map( + org.apache.paimon.rest.responses.GetPredictionResponse + ::toPrediction) + .collect(Collectors.toList()), + response.getNextPageToken()); + } catch (NoSuchResourceException e) { + throw new TableNotExistException(identifier, e); + } catch (NotImplementedException e) { + throw new UnsupportedOperationException("REST Catalog does not support predictions", e); + } + } + + @Override + public Prediction cancelPrediction(Identifier identifier, String jobId) + throws TableNotExistException, PredictionNotExistException { + try { + return api.cancelPrediction(identifier, jobId).toPrediction(); + } catch (NoSuchResourceException e) { + if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_PREDICTION)) { + throw new PredictionNotExistException(identifier, jobId, e); + } + throw new TableNotExistException(identifier, e); + } catch (NotImplementedException e) { + throw new UnsupportedOperationException("REST Catalog does not support predictions", e); + } + } + @Override public List listFunctions(String databaseName) throws DatabaseNotExistException { try { diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java new file mode 100644 index 000000000000..7651849006f5 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.PagedList; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.Prediction; +import org.apache.paimon.catalog.PredictionSpec; +import org.apache.paimon.catalog.PredictionStatus; +import org.apache.paimon.options.Options; + +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** REST catalog tests for durable asynchronous table predictions. */ +class RESTCatalogPredictionTest { + + @Test + void testPredictionLifecycleUsesSharedRestContract() throws Exception { + try (MockWebServer server = new MockWebServer()) { + server.start(); + Catalog catalog = catalog(server); + Identifier table = Identifier.create("quality", "clips"); + + server.enqueue(json(predictionJson("PENDING"), 202)); + Prediction submitted = catalog.submitPrediction(table, spec()); + assertThat(submitted.jobId()).isEqualTo("job-1"); + RecordedRequest submit = server.takeRequest(10, TimeUnit.SECONDS); + assertThat(submit.getPath()) + .isEqualTo("/v1/catalog/databases/quality/tables/clips/predictions"); + Map request = RESTApi.fromJson(submit.getBody().readUtf8(), Map.class); + assertThat(request.get("requestId")).isInstanceOf(String.class); + assertThat((String) request.get("requestId")).isNotBlank(); + + server.enqueue( + json( + "{\"predictions\":[" + + predictionJson("RUNNING") + + "],\"nextPageToken\":\"next\"}", + 200)); + PagedList page = + catalog.listPredictions( + table, "daily-quality", PredictionStatus.RUNNING, 10, "page-1"); + assertThat(page.getElements()).extracting(Prediction::jobId).containsExactly("job-1"); + assertThat(page.getNextPageToken()).isEqualTo("next"); + String listPath = server.takeRequest(10, TimeUnit.SECONDS).getPath(); + assertThat(listPath) + .contains("name=daily-quality") + .contains("status=RUNNING") + .contains("maxResults=10") + .contains("pageToken=page-1"); + + server.enqueue(json(predictionJson("CANCELLED"), 200)); + assertThat(catalog.cancelPrediction(table, "job/1").status()) + .isEqualTo(PredictionStatus.CANCELLED); + assertThat(server.takeRequest(10, TimeUnit.SECONDS).getPath()) + .isEqualTo( + "/v1/catalog/databases/quality/tables/clips/predictions/" + + "job%2F1/cancel"); + } + } + + @Test + void testPredictionNotFoundPreservesResourceType() throws Exception { + try (MockWebServer server = new MockWebServer()) { + server.start(); + Catalog catalog = catalog(server); + Identifier table = Identifier.create("quality", "clips"); + + server.enqueue(notFound("MODEL", "qwen_vl")); + assertThatThrownBy(() -> catalog.submitPrediction(table, spec())) + .isInstanceOf(Catalog.ModelNotExistException.class); + + server.enqueue(notFound("TABLE", "clips")); + assertThatThrownBy(() -> catalog.getPrediction(table, "job-1")) + .isInstanceOf(Catalog.TableNotExistException.class); + + server.enqueue(notFound("PREDICTION", "job-1")); + assertThatThrownBy(() -> catalog.getPrediction(table, "job-1")) + .isInstanceOf(Catalog.PredictionNotExistException.class); + } + } + + private static PredictionSpec spec() { + return new PredictionSpec( + "daily-quality", + "quality.qwen_vl", + Collections.singletonMap("video", "content"), + Collections.singletonList("clip_id"), + "Classify", + Collections.singletonMap("result", "STRING"), + null, + null, + null, + Collections.emptyMap()); + } + + private static Catalog catalog(MockWebServer server) { + Options options = new Options(); + options.set(RESTCatalogOptions.URI, server.url("/").toString()); + options.set(RESTCatalogOptions.TOKEN_PROVIDER, "bear"); + options.set(RESTCatalogOptions.TOKEN, "token"); + options.set(RESTCatalogInternalOptions.PREFIX, "catalog"); + return new RESTCatalog(CatalogContext.create(options), false); + } + + private static String predictionJson(String status) { + return "{\"jobId\":\"job-1\",\"name\":\"daily-quality\",\"status\":\"" + + status + + "\",\"snapshotId\":7,\"model\":\"quality.qwen_vl\"," + + "\"modelVersion\":\"1\",\"outputTable\":\"system.results\"," + + "\"createdAt\":10}"; + } + + private static MockResponse json(String body, int code) { + return new MockResponse() + .setResponseCode(code) + .setHeader("Content-Type", "application/json") + .setBody(body); + } + + private static MockResponse notFound(String resourceType, String resourceName) { + return json( + "{\"resourceType\":\"" + + resourceType + + "\",\"resourceName\":\"" + + resourceName + + "\",\"message\":\"not found\",\"code\":404}", + 404); + } +} diff --git a/paimon-python/pypaimon/api/api_response.py b/paimon-python/pypaimon/api/api_response.py index 67609852af66..5e45a9afb5c3 100644 --- a/paimon-python/pypaimon/api/api_response.py +++ b/paimon-python/pypaimon/api/api_response.py @@ -50,6 +50,7 @@ class ErrorResponse(RESTResponse): RESOURCE_TYPE_VIEW = "VIEW" RESOURCE_TYPE_FUNCTION = "FUNCTION" RESOURCE_TYPE_MODEL = "MODEL" + RESOURCE_TYPE_PREDICTION = "PREDICTION" RESOURCE_TYPE_COLUMN = "COLUMN" RESOURCE_TYPE_SNAPSHOT = "SNAPSHOT" RESOURCE_TYPE_TAG = "TAG" diff --git a/paimon-python/pypaimon/api/auth/dlf_openapi_actions.py b/paimon-python/pypaimon/api/auth/dlf_openapi_actions.py index 9491c9b32d90..fc1b99d64ef1 100644 --- a/paimon-python/pypaimon/api/auth/dlf_openapi_actions.py +++ b/paimon-python/pypaimon/api/auth/dlf_openapi_actions.py @@ -102,6 +102,10 @@ ("POST", "/v1/{prefix}/databases/{database}/models", "CreateModel"), ("GET", "/v1/{prefix}/databases/{database}/models/{model}", "GetModel"), ("DELETE", "/v1/{prefix}/databases/{database}/models/{model}", "DropModel"), + ("POST", "/v1/{prefix}/databases/{database}/tables/{table}/predictions", "SubmitPrediction"), + ("GET", "/v1/{prefix}/databases/{database}/tables/{table}/predictions", "ListPredictions"), + ("GET", "/v1/{prefix}/databases/{database}/tables/{table}/predictions/{jobId}", "GetPrediction"), + ("POST", "/v1/{prefix}/databases/{database}/tables/{table}/predictions/{jobId}/cancel", "CancelPrediction"), ("GET", "/v1/{prefix}/permissions", "ListPermissionAssignments"), ("POST", "/v1/{prefix}/permissions/grant", "GrantPermissionAssignment"), ("POST", "/v1/{prefix}/permissions/revoke", "RevokePermissionAssignment"), diff --git a/paimon-python/pypaimon/api/prediction.py b/paimon-python/pypaimon/api/prediction.py new file mode 100644 index 000000000000..232e13e991c0 --- /dev/null +++ b/paimon-python/pypaimon/api/prediction.py @@ -0,0 +1,182 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import uuid +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from pypaimon.api.api_request import RESTRequest +from pypaimon.api.api_response import PagedResponse, RESTResponse +from pypaimon.catalog.prediction import ( + Prediction, + PredictionSpec, + PredictionStatus, + _copy_string_list, + _copy_string_mapping, + _nonblank, +) +from pypaimon.common.json_util import json_field, optional_json_field + + +@dataclass +class SubmitPredictionRequest(RESTRequest): + model: str + inputs: Dict[str, str] + result_columns: List[str] = json_field("resultColumns") + prompt: str + output_schema: Dict[str, str] = json_field("outputSchema") + name: Optional[str] = optional_json_field("name", "non_null") + request_id: Optional[str] = optional_json_field("requestId", "non_null") + where: Optional[str] = optional_json_field("where", "non_null") + snapshot_id: Optional[int] = optional_json_field("snapshotId", "non_null") + tag_name: Optional[str] = optional_json_field("tagName", "non_null") + options: Dict[str, Any] = field(default_factory=dict) + + def __post_init__(self): + for value, field_name in ( + (self.model, "model"), + (self.prompt, "prompt"), + ): + _nonblank(value, field_name) + for value, field_name in ( + (self.name, "name"), + (self.where, "where"), + (self.tag_name, "tag_name"), + ): + if value is not None: + _nonblank(value, field_name) + if self.request_id is None: + self.request_id = uuid.uuid4().hex + else: + _nonblank(self.request_id, "request_id") + if self.snapshot_id is not None and ( + type(self.snapshot_id) is not int or self.snapshot_id <= 0): + raise ValueError("snapshot_id must be a positive integer") + if self.snapshot_id is not None and self.tag_name is not None: + raise ValueError("snapshot_id and tag_name are mutually exclusive") + self.inputs = _copy_string_mapping(self.inputs, "inputs", True) + self.result_columns = _copy_string_list( + self.result_columns, "result_columns") + self.output_schema = _copy_string_mapping( + self.output_schema, "output_schema", True) + if not isinstance(self.options, dict): + raise ValueError("options must be a mapping") + self.options = dict(self.options) + + @classmethod + def from_spec(cls, spec: PredictionSpec) -> "SubmitPredictionRequest": + return cls( + name=spec.name, + model=spec.model, + inputs=spec.inputs, + result_columns=spec.result_columns, + prompt=spec.prompt, + output_schema=spec.output_schema, + where=spec.where, + snapshot_id=spec.snapshot_id, + tag_name=spec.tag_name, + options=spec.options, + ) + + +@dataclass +class CancelPredictionRequest(RESTRequest): + """Empty body for a prediction cancellation request.""" + + pass + + +@dataclass +class GetPredictionResponse(RESTResponse): + job_id: str = json_field("jobId") + status: str = "" + snapshot_id: int = json_field("snapshotId", default=0) + model: str = "" + model_version: str = json_field("modelVersion", default="") + output_table: str = json_field("outputTable", default="") + created_at: int = json_field("createdAt", default=0) + name: Optional[str] = optional_json_field("name", "non_null") + started_at: Optional[int] = optional_json_field("startedAt", "non_null") + finished_at: Optional[int] = optional_json_field("finishedAt", "non_null") + error: Optional[str] = optional_json_field("error", "non_null") + total: Optional[int] = optional_json_field("total", "non_null") + succeeded: Optional[int] = optional_json_field("succeeded", "non_null") + failed: Optional[int] = optional_json_field("failed", "non_null") + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "GetPredictionResponse": + return cls( + job_id=data.get("jobId"), + name=data.get("name"), + status=data.get("status"), + snapshot_id=data.get("snapshotId"), + model=data.get("model"), + model_version=data.get("modelVersion"), + output_table=data.get("outputTable"), + created_at=data.get("createdAt"), + started_at=data.get("startedAt"), + finished_at=data.get("finishedAt"), + error=data.get("error"), + total=data.get("total"), + succeeded=data.get("succeeded"), + failed=data.get("failed"), + ) + + def to_prediction(self) -> Prediction: + return Prediction( + job_id=self.job_id, + name=self.name, + status=PredictionStatus(self.status), + snapshot_id=self.snapshot_id, + model=self.model, + model_version=self.model_version, + output_table=self.output_table, + created_at=self.created_at, + started_at=self.started_at, + finished_at=self.finished_at, + error=self.error, + total=self.total, + succeeded=self.succeeded, + failed=self.failed, + ) + + +@dataclass +class ListPredictionsResponse(PagedResponse[GetPredictionResponse]): + predictions: List[GetPredictionResponse] = field(default_factory=list) + next_page_token: Optional[str] = json_field("nextPageToken", default=None) + + def __post_init__(self): + if self.predictions is None: + self.predictions = [] + + def data(self) -> List[GetPredictionResponse]: + return self.predictions + + def get_next_page_token(self) -> Optional[str]: + return self.next_page_token + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "ListPredictionsResponse": + return cls( + predictions=[ + GetPredictionResponse.from_dict(value) + if isinstance(value, dict) else value + for value in (data.get("predictions") or []) + ], + next_page_token=data.get("nextPageToken"), + ) diff --git a/paimon-python/pypaimon/api/resource_paths.py b/paimon-python/pypaimon/api/resource_paths.py index 807744132aee..c7f37d852bad 100644 --- a/paimon-python/pypaimon/api/resource_paths.py +++ b/paimon-python/pypaimon/api/resource_paths.py @@ -33,6 +33,7 @@ class ResourcePaths: FUNCTIONS = "functions" FUNCTION_DETAILS = "function-details" MODELS = "models" + PREDICTIONS = "predictions" TAGS = "tags" BRANCHES = "branches" FORWARD = "forward" @@ -132,6 +133,21 @@ def model(self, database_name: str, model_name: str) -> str: self.models(database_name), RESTUtil.encode_string(model_name) ) + def predictions(self, database_name: str, table_name: str) -> str: + return "{}/{}".format( + self.table(database_name, table_name), self.PREDICTIONS) + + def prediction(self, database_name: str, table_name: str, job_id: str) -> str: + if not isinstance(job_id, str) or not job_id.strip(): + raise ValueError("job_id must be a non-empty string") + return "{}/{}".format( + self.predictions(database_name, table_name), RESTUtil.encode_string(job_id)) + + def cancel_prediction( + self, database_name: str, table_name: str, job_id: str) -> str: + return "{}/cancel".format( + self.prediction(database_name, table_name, job_id)) + def tags(self, database_name: str, table_name: str) -> str: return "{}/{}/{}/{}/{}/{}".format( self.base_path, self.DATABASES, RESTUtil.encode_string(database_name), diff --git a/paimon-python/pypaimon/api/rest_api.py b/paimon-python/pypaimon/api/rest_api.py index cdd608f29cde..0cfbdbc71ef8 100755 --- a/paimon-python/pypaimon/api/rest_api.py +++ b/paimon-python/pypaimon/api/rest_api.py @@ -53,6 +53,10 @@ from pypaimon.api.auth import AuthProviderFactory, RESTAuthFunction from pypaimon.api.client import HttpClient from pypaimon.api.model import CreateModelRequest, GetModelResponse, ListModelsResponse +from pypaimon.api.prediction import ( + CancelPredictionRequest, GetPredictionResponse, ListPredictionsResponse, + SubmitPredictionRequest, +) from pypaimon.api.resource_paths import ResourcePaths from pypaimon.api.rest_exception import NoSuchResourceException from pypaimon.api.rest_util import RESTUtil @@ -61,6 +65,7 @@ from pypaimon.common.options import Options from pypaimon.common.options.config import CatalogOptions from pypaimon.common.identifier import Identifier +from pypaimon.catalog.prediction import PredictionStatus from pypaimon.management.data_policy import DataPolicy from pypaimon.management.list_permissions_request import \ ListPermissionsRequest @@ -785,6 +790,64 @@ def drop_model(self, identifier: Identifier) -> None: self.rest_auth_function, ) + def submit_prediction( + self, + identifier: Identifier, + request: SubmitPredictionRequest, + ) -> GetPredictionResponse: + database_name, table_name = self._validate_resource_path(identifier) + return self.client.post_with_response_type( + self.resource_paths.predictions(database_name, table_name), + request, + GetPredictionResponse, + self.rest_auth_function, + ) + + def get_prediction( + self, identifier: Identifier, job_id: str + ) -> GetPredictionResponse: + database_name, table_name = self._validate_resource_path(identifier) + return self.client.get( + self.resource_paths.prediction(database_name, table_name, job_id), + GetPredictionResponse, + self.rest_auth_function, + ) + + def list_predictions( + self, + identifier: Identifier, + name: Optional[str] = None, + status: Optional[PredictionStatus] = None, + max_results: Optional[int] = None, + page_token: Optional[str] = None, + ) -> PagedList[GetPredictionResponse]: + database_name, table_name = self._validate_resource_path(identifier) + query_params = {} + self._put_query_parameter(query_params, "name", name) + self._put_query_parameter( + query_params, "status", status.value if status is not None else None) + if max_results is not None and max_results > 0: + query_params[self.MAX_RESULTS] = str(max_results) + self._put_query_parameter(query_params, self.PAGE_TOKEN, page_token) + response = self.client.get_with_params( + self.resource_paths.predictions(database_name, table_name), + query_params, + ListPredictionsResponse, + self.rest_auth_function, + ) + return PagedList(response.predictions, response.next_page_token) + + def cancel_prediction( + self, identifier: Identifier, job_id: str + ) -> GetPredictionResponse: + database_name, table_name = self._validate_resource_path(identifier) + return self.client.post_with_response_type( + self.resource_paths.cancel_prediction(database_name, table_name, job_id), + CancelPredictionRequest(), + GetPredictionResponse, + self.rest_auth_function, + ) + def auth_table_query(self, identifier: Identifier, select: Optional[List[str]]) -> AuthTableQueryResponse: database_name, table_name = self.__validate_identifier(identifier) request = AuthTableQueryRequest(select=select) diff --git a/paimon-python/pypaimon/catalog/catalog.py b/paimon-python/pypaimon/catalog/catalog.py index 9ea6a83074a0..62436dee2f22 100644 --- a/paimon-python/pypaimon/catalog/catalog.py +++ b/paimon-python/pypaimon/catalog/catalog.py @@ -111,6 +111,45 @@ def drop_model( """ raise NotImplementedError("drop_model is not supported by this catalog.") + def submit_prediction( + self, + identifier: Union[str, Identifier], + spec: 'PredictionSpec', + ) -> 'Prediction': + """Submit an asynchronous prediction over a table snapshot.""" + raise NotImplementedError( + "submit_prediction is not supported by this catalog.") + + def get_prediction( + self, + identifier: Union[str, Identifier], + job_id: str, + ) -> 'Prediction': + """Get one prediction by its unique job id.""" + raise NotImplementedError( + "get_prediction is not supported by this catalog.") + + def list_predictions( + self, + identifier: Union[str, Identifier], + name: Optional[str] = None, + status: Optional['PredictionStatus'] = None, + max_results: Optional[int] = None, + page_token: Optional[str] = None, + ) -> PagedList['Prediction']: + """List predictions for a table, optionally filtered by name or status.""" + raise NotImplementedError( + "list_predictions is not supported by this catalog.") + + def cancel_prediction( + self, + identifier: Union[str, Identifier], + job_id: str, + ) -> 'Prediction': + """Request cancellation and return the updated prediction.""" + raise NotImplementedError( + "cancel_prediction is not supported by this catalog.") + def alter_database(self, name: str, changes: List[PropertyChange]) -> None: """Alter database properties. diff --git a/paimon-python/pypaimon/catalog/catalog_exception.py b/paimon-python/pypaimon/catalog/catalog_exception.py index cb2105401157..b0d104b4b9ec 100644 --- a/paimon-python/pypaimon/catalog/catalog_exception.py +++ b/paimon-python/pypaimon/catalog/catalog_exception.py @@ -135,6 +135,16 @@ def __init__(self, identifier: Identifier): super().__init__(f"Model {identifier.get_full_name()} already exists") +class PredictionNotExistException(CatalogException): + """Prediction not exist exception""" + + def __init__(self, identifier: Identifier, job_id: str): + self.identifier = identifier + self.job_id = job_id + super().__init__( + f"Prediction {job_id} for table {identifier.get_full_name()} does not exist") + + class ColumnNotExistException(CatalogException): """Column not exist exception""" diff --git a/paimon-python/pypaimon/catalog/prediction.py b/paimon-python/pypaimon/catalog/prediction.py new file mode 100644 index 000000000000..83aa00915551 --- /dev/null +++ b/paimon-python/pypaimon/catalog/prediction.py @@ -0,0 +1,146 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Dict, List, Optional + + +class PredictionStatus(str, Enum): + PENDING = "PENDING" + RUNNING = "RUNNING" + SUCCEEDED = "SUCCEEDED" + PARTIAL_SUCCESS = "PARTIAL_SUCCESS" + FAILED = "FAILED" + CANCELLED = "CANCELLED" + + @property + def terminal(self) -> bool: + return self in { + PredictionStatus.SUCCEEDED, + PredictionStatus.PARTIAL_SUCCESS, + PredictionStatus.FAILED, + PredictionStatus.CANCELLED, + } + + +@dataclass +class PredictionSpec: + """Provider-neutral input for an asynchronous table prediction.""" + + model: str + inputs: Dict[str, str] + result_columns: List[str] + prompt: str + output_schema: Dict[str, str] + name: Optional[str] = None + where: Optional[str] = None + snapshot_id: Optional[int] = None + tag_name: Optional[str] = None + options: Dict[str, Any] = field(default_factory=dict) + + def __post_init__(self): + _nonblank(self.model, "model") + _nonblank(self.prompt, "prompt") + for value, name in ( + (self.name, "name"), + (self.where, "where"), + (self.tag_name, "tag_name"), + ): + if value is not None: + _nonblank(value, name) + if self.snapshot_id is not None and ( + type(self.snapshot_id) is not int or self.snapshot_id <= 0): + raise ValueError("snapshot_id must be a positive integer") + if self.snapshot_id is not None and self.tag_name is not None: + raise ValueError("snapshot_id and tag_name are mutually exclusive") + self.inputs = _copy_string_mapping(self.inputs, "inputs", True) + self.result_columns = _copy_string_list( + self.result_columns, "result_columns") + self.output_schema = _copy_string_mapping( + self.output_schema, "output_schema", True) + if not isinstance(self.options, dict): + raise ValueError("options must be a mapping") + self.options = dict(self.options) + + +@dataclass +class Prediction: + """Durable metadata for one asynchronous table prediction.""" + + job_id: str + status: PredictionStatus + snapshot_id: int + model: str + model_version: str + output_table: str + created_at: int + name: Optional[str] = None + started_at: Optional[int] = None + finished_at: Optional[int] = None + error: Optional[str] = None + total: Optional[int] = None + succeeded: Optional[int] = None + failed: Optional[int] = None + + def __post_init__(self): + _nonblank(self.job_id, "job_id") + if isinstance(self.status, str): + self.status = PredictionStatus(self.status) + if not isinstance(self.status, PredictionStatus): + raise ValueError("status must be a PredictionStatus") + if type(self.snapshot_id) is not int or self.snapshot_id <= 0: + raise ValueError("snapshot_id must be a positive integer") + _nonblank(self.model, "model") + _nonblank(self.model_version, "model_version") + _nonblank(self.output_table, "output_table") + if type(self.created_at) is not int or self.created_at < 0: + raise ValueError("created_at must be a non-negative integer") + for name in ( + "started_at", "finished_at", "total", "succeeded", "failed"): + value = getattr(self, name) + if value is not None and (type(value) is not int or value < 0): + raise ValueError( + "{} must be a non-negative integer".format(name)) + + +def _nonblank(value, name): + if not isinstance(value, str) or not value.strip(): + raise ValueError("{} must be a non-empty string".format(name)) + + +def _copy_string_mapping(values, name, required): + if not isinstance(values, dict) or (required and not values): + raise ValueError("{} must be a non-empty mapping".format(name)) + result = {} + for key, value in values.items(): + _nonblank(key, "{} key".format(name)) + _nonblank(value, "{} value".format(name)) + result[key] = value + return result + + +def _copy_string_list(values, name): + if not isinstance(values, list): + raise ValueError("{} must be a list".format(name)) + result = [] + for value in values: + _nonblank(value, "{} value".format(name)) + if value in result: + raise ValueError("{} must not contain duplicates".format(name)) + result.append(value) + return result diff --git a/paimon-python/pypaimon/catalog/rest/rest_catalog.py b/paimon-python/pypaimon/catalog/rest/rest_catalog.py index 6caf66564a05..e453bdcca9f0 100644 --- a/paimon-python/pypaimon/catalog/rest/rest_catalog.py +++ b/paimon-python/pypaimon/catalog/rest/rest_catalog.py @@ -19,6 +19,7 @@ from typing import Any, Callable, Dict, List, Optional, Union from pypaimon.api.api_response import ErrorResponse, GetTableResponse, GetTagResponse, PagedList, Partition from pypaimon.api.model import CreateModelRequest +from pypaimon.api.prediction import SubmitPredictionRequest from pypaimon.api.rest_api import RESTApi from pypaimon.api.rest_permission_management import RESTPermissionManagement from pypaimon.api.rest_policy_management import RESTPolicyManagement @@ -38,12 +39,14 @@ TableIdNotExistException, TableIdNoPermissionException, FunctionNotExistException, FunctionAlreadyExistException, ModelNotExistException, ModelAlreadyExistException, + PredictionNotExistException, DefinitionAlreadyExistException, DefinitionNotExistException, TagNotExistException, TagAlreadyExistException, BranchNotExistException, BranchAlreadyExistException, ) from pypaimon.catalog.database import Database from pypaimon.catalog.model import Model, ModelDefinition +from pypaimon.catalog.prediction import Prediction, PredictionSpec, PredictionStatus from pypaimon.catalog.rest.property_change import PropertyChange from pypaimon.catalog.rest.rest_token_file_io import RESTTokenFileIO from pypaimon.catalog.rest.table_metadata import TableMetadata @@ -678,6 +681,84 @@ def drop_model( except NotImplementedException as e: raise NotImplementedError("REST Catalog does not support models") from e + def submit_prediction( + self, + identifier: Union[str, Identifier], + spec: PredictionSpec, + ) -> Prediction: + if not isinstance(identifier, Identifier): + identifier = Identifier.from_string(identifier) + check_not_branch(identifier, "submit_prediction") + check_not_system_table(identifier, "submit_prediction") + try: + return self.rest_api.submit_prediction( + identifier, SubmitPredictionRequest.from_spec(spec)).to_prediction() + except NoSuchResourceException as e: + if e.resource_type == ErrorResponse.RESOURCE_TYPE_MODEL: + raise ModelNotExistException(Identifier.from_string(spec.model)) from e + raise TableNotExistException(identifier) from e + except NotImplementedException as e: + raise NotImplementedError( + "REST Catalog does not support predictions") from e + + def get_prediction( + self, + identifier: Union[str, Identifier], + job_id: str, + ) -> Prediction: + if not isinstance(identifier, Identifier): + identifier = Identifier.from_string(identifier) + try: + return self.rest_api.get_prediction(identifier, job_id).to_prediction() + except NoSuchResourceException as e: + if e.resource_type == ErrorResponse.RESOURCE_TYPE_PREDICTION: + raise PredictionNotExistException(identifier, job_id) from e + raise TableNotExistException(identifier) from e + except NotImplementedException as e: + raise NotImplementedError( + "REST Catalog does not support predictions") from e + + def list_predictions( + self, + identifier: Union[str, Identifier], + name: Optional[str] = None, + status: Optional[PredictionStatus] = None, + max_results: Optional[int] = None, + page_token: Optional[str] = None, + ) -> PagedList[Prediction]: + if not isinstance(identifier, Identifier): + identifier = Identifier.from_string(identifier) + try: + page = self.rest_api.list_predictions( + identifier, name, status, max_results, page_token) + return PagedList( + [value.to_prediction() for value in page.elements], + page.next_page_token, + ) + except NoSuchResourceException as e: + raise TableNotExistException(identifier) from e + except NotImplementedException as e: + raise NotImplementedError( + "REST Catalog does not support predictions") from e + + def cancel_prediction( + self, + identifier: Union[str, Identifier], + job_id: str, + ) -> Prediction: + if not isinstance(identifier, Identifier): + identifier = Identifier.from_string(identifier) + try: + return self.rest_api.cancel_prediction( + identifier, job_id).to_prediction() + except NoSuchResourceException as e: + if e.resource_type == ErrorResponse.RESOURCE_TYPE_PREDICTION: + raise PredictionNotExistException(identifier, job_id) from e + raise TableNotExistException(identifier) from e + except NotImplementedException as e: + raise NotImplementedError( + "REST Catalog does not support predictions") from e + # Tag CRUD: mirrors Java RESTCatalog tag handlers. def create_tag(self, identifier: Union[str, Identifier], tag_name: str, snapshot_id: Optional[int] = None, diff --git a/paimon-python/pypaimon/multimodal/prediction.py b/paimon-python/pypaimon/multimodal/prediction.py new file mode 100644 index 000000000000..6a82e2b300fa --- /dev/null +++ b/paimon-python/pypaimon/multimodal/prediction.py @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import time +from datetime import timedelta +from typing import Optional + +from pypaimon.catalog.prediction import Prediction +from pypaimon.common.identifier import Identifier + + +class PredictionJob: + """Client handle for a durable asynchronous table prediction.""" + + def __init__( + self, catalog, identifier: Identifier, prediction: Prediction): + self._catalog = catalog + self._identifier = identifier + self._prediction = prediction + + @property + def prediction(self) -> Prediction: + return self._prediction + + def __getattr__(self, name): + return getattr(self._prediction, name) + + def refresh(self) -> 'PredictionJob': + self._prediction = self._catalog.get_prediction( + self._identifier, self._prediction.job_id) + return self + + def wait( + self, + timeout: Optional[timedelta] = None, + poll_interval: float = 1.0, + ) -> 'PredictionJob': + """Poll prediction metadata until it reaches a terminal state. + + This is client-side polling over ``get_prediction``. It does not keep + a long-lived request open on the REST service. + """ + if timeout is not None and not isinstance(timeout, timedelta): + raise ValueError("timeout must be a datetime.timedelta") + if poll_interval < 0: + raise ValueError("poll_interval must not be negative") + deadline = ( + None if timeout is None + else time.monotonic() + timeout.total_seconds() + ) + while not self._prediction.status.terminal: + if deadline is not None: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError( + "Prediction {} did not finish within {}".format( + self._prediction.job_id, timeout)) + time.sleep(min(poll_interval, remaining)) + else: + time.sleep(poll_interval) + self.refresh() + return self + + def cancel(self) -> 'PredictionJob': + self._prediction = self._catalog.cancel_prediction( + self._identifier, self._prediction.job_id) + return self diff --git a/paimon-python/pypaimon/multimodal/table.py b/paimon-python/pypaimon/multimodal/table.py index de39dcb5be20..5523c84152cc 100644 --- a/paimon-python/pypaimon/multimodal/table.py +++ b/paimon-python/pypaimon/multimodal/table.py @@ -111,6 +111,90 @@ def copy(self, options: dict) -> 'MultimodalTable': return MultimodalTable( self.catalog, raw_table.identifier.get_full_name(), raw_table) + def predict( + self, + *, + model, + inputs, + result_columns, + prompt, + output_schema, + name=None, + where=None, + options=None): + """Submit an asynchronous prediction without reading rows locally.""" + from pypaimon.catalog.prediction import PredictionSpec + from pypaimon.multimodal.prediction import PredictionJob + from pypaimon.table.file_store_table import FileStoreTable + + if not isinstance(self.raw_table, FileStoreTable): + raise NotImplementedError( + "Prediction requires a Paimon file store table") + table_options = self.raw_table.options.options.to_map() + allowed = {"scan.snapshot-id", "scan.tag-name"} + unsupported = [ + key for key in table_options + if (key.startswith("scan.") or key.startswith("incremental-")) + and key not in allowed + ] + if unsupported: + raise ValueError( + "Prediction does not support scan options: {}".format(unsupported)) + spec = PredictionSpec( + name=name, + model=model, + inputs=inputs, + result_columns=result_columns, + prompt=prompt, + output_schema=output_schema, + where=where, + snapshot_id=self.raw_table.options.scan_snapshot_id(), + tag_name=self.raw_table.options.scan_tag_name(), + options=options or {}, + ) + prediction = self.catalog.submit_prediction( + self.raw_table.identifier, spec) + return PredictionJob( + self.catalog, self.raw_table.identifier, prediction) + + def get_prediction(self, job_id): + """Return a client handle for one asynchronous prediction.""" + from pypaimon.multimodal.prediction import PredictionJob + + prediction = self.catalog.get_prediction( + self.raw_table.identifier, job_id) + return PredictionJob( + self.catalog, self.raw_table.identifier, prediction) + + def list_predictions( + self, + *, + name=None, + status=None, + max_results=None, + page_token=None): + """List prediction handles for this table.""" + from pypaimon.api.api_response import PagedList + from pypaimon.catalog.prediction import PredictionStatus + from pypaimon.multimodal.prediction import PredictionJob + + if isinstance(status, str): + status = PredictionStatus(status) + page = self.catalog.list_predictions( + self.raw_table.identifier, + name=name, + status=status, + max_results=max_results, + page_token=page_token, + ) + return PagedList( + [ + PredictionJob(self.catalog, self.raw_table.identifier, value) + for value in page.elements + ], + page.next_page_token, + ) + def add(self, data): arrow_table = _to_arrow_table(data, _target_schema(self.raw_table)) write_builder = self.raw_table.new_batch_write_builder() diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py new file mode 100644 index 000000000000..f699c839fe2b --- /dev/null +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -0,0 +1,242 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +import threading +import unittest +from datetime import timedelta +from http.server import BaseHTTPRequestHandler, HTTPServer +from unittest.mock import Mock + +from pypaimon.api.prediction import ( + GetPredictionResponse, + ListPredictionsResponse, + SubmitPredictionRequest, +) +from pypaimon.api.rest_api import RESTApi +from pypaimon.catalog.prediction import ( + Prediction, + PredictionSpec, + PredictionStatus, +) +from pypaimon.catalog.rest.rest_catalog import RESTCatalog +from pypaimon.common.identifier import Identifier +from pypaimon.common.json_util import JSON +from pypaimon.common.options.core_options import CoreOptions +from pypaimon.common.options.options import Options +from pypaimon.multimodal.prediction import PredictionJob +from pypaimon.multimodal.table import MultimodalTable +from pypaimon.table.file_store_table import FileStoreTable + + +def _spec(**changes): + values = dict( + model="quality.qwen_vl", + inputs={"video": "content"}, + result_columns=["clip_id"], + prompt="Classify", + output_schema={"result": "STRING"}, + ) + values.update(changes) + return PredictionSpec(**values) + + +def _prediction(status=PredictionStatus.PENDING, **changes): + values = dict( + job_id="job-1", + name="daily-quality", + status=status, + snapshot_id=7, + model="quality.qwen_vl", + model_version="1", + output_table="system.prediction_results", + created_at=10, + ) + values.update(changes) + return Prediction(**values) + + +def _response(status="PENDING", **changes): + values = dict( + job_id="job-1", + name="daily-quality", + status=status, + snapshot_id=7, + model="quality.qwen_vl", + model_version="1", + output_table="system.prediction_results", + created_at=10, + ) + values.update(changes) + return GetPredictionResponse(**values) + + +class PredictionTest(unittest.TestCase): + + def test_request_and_response_wire_format(self): + request = SubmitPredictionRequest.from_spec(_spec()) + body = json.loads(JSON.to_json(request)) + self.assertTrue(body["requestId"]) + self.assertEqual(["clip_id"], body["resultColumns"]) + self.assertEqual({"result": "STRING"}, body["outputSchema"]) + self.assertNotIn("snapshotId", body) + + response = JSON.from_json( + JSON.to_json(_response()), GetPredictionResponse) + self.assertEqual( + PredictionStatus.PENDING, response.to_prediction().status) + page = JSON.from_json( + JSON.to_json(ListPredictionsResponse([_response()], "next")), + ListPredictionsResponse, + ) + self.assertEqual("job-1", page.predictions[0].job_id) + self.assertEqual("next", page.next_page_token) + + def test_validation(self): + invalid = [ + dict(model=""), + dict(inputs={}), + dict(inputs={"video": ""}), + dict(result_columns=["id", "id"]), + dict(result_columns="id"), + dict(prompt=" "), + dict(output_schema={}), + dict(snapshot_id=0), + dict(snapshot_id=1, tag_name="v1"), + ] + for changes in invalid: + with self.subTest(changes=changes), self.assertRaises(ValueError): + _spec(**changes) + + def test_table_submit_fixes_view_and_returns_job(self): + raw = Mock(spec=FileStoreTable) + raw.options = CoreOptions(Options({"scan.snapshot-id": "7"})) + raw.identifier = Identifier.create("quality", "clips") + catalog = Mock() + catalog.submit_prediction.return_value = _prediction() + table = MultimodalTable(catalog, "quality.clips", raw) + + job = table.predict( + name="daily-quality", + model="quality.qwen_vl", + inputs={"video": "content"}, + result_columns=["clip_id"], + prompt="Classify", + output_schema={"result": "STRING"}, + ) + + self.assertIsInstance(job, PredictionJob) + self.assertEqual("job-1", job.job_id) + sent = catalog.submit_prediction.call_args.args[1] + self.assertEqual(7, sent.snapshot_id) + self.assertIsNone(sent.tag_name) + raw.new_read_builder.assert_not_called() + + def test_wait_is_client_side_polling(self): + catalog = Mock() + catalog.get_prediction.side_effect = [ + _prediction(PredictionStatus.RUNNING), + _prediction( + PredictionStatus.SUCCEEDED, total=2, succeeded=2, failed=0), + ] + job = PredictionJob( + catalog, Identifier.create("quality", "clips"), _prediction()) + result = job.wait(timeout=timedelta(seconds=1), poll_interval=0) + self.assertIs(job, result) + self.assertEqual(PredictionStatus.SUCCEEDED, job.status) + self.assertEqual(2, catalog.get_prediction.call_count) + + def test_rest_round_trip_paths_and_filters(self): + calls = [] + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def _send(self, payload, status=200): + body = JSON.to_json(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_POST(self): + length = int(self.headers.get("Content-Length", "0")) + body = json.loads(self.rfile.read(length)) if length else None + calls.append(("POST", self.path, body)) + status = 202 if self.path.endswith("/predictions") else 200 + self._send(_response(), status) + + def do_GET(self): + calls.append(("GET", self.path, None)) + if "/predictions?" in self.path: + self._send(ListPredictionsResponse([_response()], "next")) + else: + self._send(_response()) + + server = HTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + api = RESTApi( + { + "uri": "http://127.0.0.1:%s" % server.server_port, + "token.provider": "bear", + "token": "test", + "prefix": "catalog", + }, + False, + ) + catalog = object.__new__(RESTCatalog) + catalog.rest_api = api + identifier = Identifier.create("a/b", "clip table") + + self.assertEqual("job-1", catalog.submit_prediction( + identifier, _spec()).job_id) + self.assertEqual( + "/v1/catalog/databases/a%2Fb/tables/clip+table/predictions", + calls[-1][1], + ) + self.assertTrue(calls[-1][2]["requestId"]) + + page = catalog.list_predictions( + identifier, + name="daily-quality", + status=PredictionStatus.RUNNING, + max_results=10, + page_token="p1", + ) + self.assertEqual("job-1", page.elements[0].job_id) + self.assertIn("name=daily-quality", calls[-1][1]) + self.assertIn("status=RUNNING", calls[-1][1]) + self.assertIn("maxResults=10", calls[-1][1]) + self.assertIn("pageToken=p1", calls[-1][1]) + + catalog.get_prediction(identifier, "job/1") + self.assertTrue(calls[-1][1].endswith("/predictions/job%2F1")) + catalog.cancel_prediction(identifier, "job/1") + self.assertTrue( + calls[-1][1].endswith("/predictions/job%2F1/cancel")) + finally: + server.shutdown() + server.server_close() + thread.join() + + +if __name__ == "__main__": + unittest.main() diff --git a/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py b/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py index 72cbc13e3bf4..a20920797c2b 100644 --- a/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py +++ b/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py @@ -27,6 +27,7 @@ from pypaimon.api.rest_api import RESTApi from pypaimon.api.model import CreateModelRequest +from pypaimon.api.prediction import SubmitPredictionRequest from pypaimon.catalog.model import ModelField, ProviderModelDefinition from pypaimon.common.identifier import Identifier from pypaimon.function.function import FunctionImpl @@ -63,6 +64,13 @@ [ModelField("video", "BLOB", "video")], ), ), + SubmitPredictionRequest: SubmitPredictionRequest( + model="db.qwen_vl", + inputs={"video": "content"}, + result_columns=["id"], + prompt="classify", + output_schema={"result": "STRING"}, + ), str: "SELECT", bool: False, Identifier: Identifier.create("db", "t"), From a5ddb2f44b1779419fea8affe64c3dffb6d2c8a9 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Mon, 16 Nov 2026 18:02:13 +0800 Subject: [PATCH 02/41] [core][python] Clarify prediction idempotency --- docs/static/rest-catalog-open-api.yaml | 8 +++++--- .../rest/requests/SubmitPredictionRequest.java | 18 ++++++++++-------- .../paimon/rest/RESTApiPredictionTest.java | 6 ++++-- .../paimon/rest/RESTCatalogPredictionTest.java | 4 ++-- paimon-python/pypaimon/api/prediction.py | 9 +++++---- .../pypaimon/tests/rest/prediction_test.py | 8 ++++++-- 6 files changed, 32 insertions(+), 21 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 1fd26e7f3737..87f264768e87 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -140,6 +140,8 @@ paths: operationId: submitPrediction description: >- Submits an asynchronous prediction with a fixed snapshot and model version. + Reusing an idempotencyKey for the same table and payload returns the existing + job; reusing it with a different payload returns 409. parameters: - {name: prefix, in: path, required: true, schema: {type: string}} - {name: database, in: path, required: true, schema: {type: string}} @@ -4681,16 +4683,16 @@ components: SubmitPredictionRequest: type: object additionalProperties: false - required: [requestId, model, inputs, resultColumns, prompt, outputSchema] + required: [idempotencyKey, model, inputs, resultColumns, prompt, outputSchema] properties: name: type: string minLength: 1 description: Optional non-unique label. - requestId: + idempotencyKey: type: string minLength: 1 - description: Client-generated idempotency key. + description: SDK-generated key reused for retries of one logical submission. model: type: string minLength: 1 diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java index 88e03e680c16..c0d4915a63b9 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java @@ -40,7 +40,7 @@ public class SubmitPredictionRequest implements RESTRequest { private static final String FIELD_NAME = "name"; - private static final String FIELD_REQUEST_ID = "requestId"; + private static final String FIELD_IDEMPOTENCY_KEY = "idempotencyKey"; private static final String FIELD_MODEL = "model"; private static final String FIELD_INPUTS = "inputs"; private static final String FIELD_RESULT_COLUMNS = "resultColumns"; @@ -51,13 +51,13 @@ public class SubmitPredictionRequest implements RESTRequest { private static final String FIELD_TAG_NAME = "tagName"; private static final String FIELD_OPTIONS = "options"; - private final String requestId; + private final String idempotencyKey; private final PredictionSpec spec; @JsonCreator public SubmitPredictionRequest( @Nullable @JsonProperty(FIELD_NAME) String name, - @Nullable @JsonProperty(FIELD_REQUEST_ID) String requestId, + @Nullable @JsonProperty(FIELD_IDEMPOTENCY_KEY) String idempotencyKey, @JsonProperty(FIELD_MODEL) String model, @JsonProperty(FIELD_INPUTS) Map inputs, @JsonProperty(FIELD_RESULT_COLUMNS) List resultColumns, @@ -68,8 +68,10 @@ public SubmitPredictionRequest( @Nullable @JsonProperty(FIELD_TAG_NAME) String tagName, @Nullable @JsonProperty(FIELD_OPTIONS) Map options) { checkArgument( - requestId == null || !requestId.trim().isEmpty(), "requestId must not be blank"); - this.requestId = requestId == null ? UUID.randomUUID().toString() : requestId; + idempotencyKey == null || !idempotencyKey.trim().isEmpty(), + "idempotencyKey must not be blank"); + this.idempotencyKey = + idempotencyKey == null ? UUID.randomUUID().toString() : idempotencyKey; this.spec = new PredictionSpec( name, @@ -106,9 +108,9 @@ public String getName() { return spec.name(); } - @JsonGetter(FIELD_REQUEST_ID) - public String getRequestId() { - return requestId; + @JsonGetter(FIELD_IDEMPOTENCY_KEY) + public String getIdempotencyKey() { + return idempotencyKey; } @JsonGetter(FIELD_MODEL) diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java index 8ceb09f1edb8..8a1663aa8601 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java @@ -42,8 +42,10 @@ class RESTApiPredictionTest { void testRequestAndResponseWireFormat() throws Exception { SubmitPredictionRequest request = new SubmitPredictionRequest(spec()); Map requestMap = RESTApi.fromJson(RESTApi.toJson(request), Map.class); - assertThat(requestMap.get("requestId")).isInstanceOf(String.class); - assertThat((String) requestMap.get("requestId")).isNotBlank(); + assertThat(requestMap.get("idempotencyKey")).isInstanceOf(String.class); + assertThat((String) requestMap.get("idempotencyKey")).isNotBlank(); + assertThat(RESTApi.fromJson(RESTApi.toJson(request), Map.class).get("idempotencyKey")) + .isEqualTo(requestMap.get("idempotencyKey")); assertThat(requestMap) .containsEntry("model", "quality.qwen_vl") .doesNotContainKeys("name", "where", "snapshotId", "tagName"); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java index 7651849006f5..c6ced782abee 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java @@ -56,8 +56,8 @@ void testPredictionLifecycleUsesSharedRestContract() throws Exception { assertThat(submit.getPath()) .isEqualTo("/v1/catalog/databases/quality/tables/clips/predictions"); Map request = RESTApi.fromJson(submit.getBody().readUtf8(), Map.class); - assertThat(request.get("requestId")).isInstanceOf(String.class); - assertThat((String) request.get("requestId")).isNotBlank(); + assertThat(request.get("idempotencyKey")).isInstanceOf(String.class); + assertThat((String) request.get("idempotencyKey")).isNotBlank(); server.enqueue( json( diff --git a/paimon-python/pypaimon/api/prediction.py b/paimon-python/pypaimon/api/prediction.py index 232e13e991c0..40aa09158a2f 100644 --- a/paimon-python/pypaimon/api/prediction.py +++ b/paimon-python/pypaimon/api/prediction.py @@ -40,7 +40,8 @@ class SubmitPredictionRequest(RESTRequest): prompt: str output_schema: Dict[str, str] = json_field("outputSchema") name: Optional[str] = optional_json_field("name", "non_null") - request_id: Optional[str] = optional_json_field("requestId", "non_null") + idempotency_key: Optional[str] = optional_json_field( + "idempotencyKey", "non_null") where: Optional[str] = optional_json_field("where", "non_null") snapshot_id: Optional[int] = optional_json_field("snapshotId", "non_null") tag_name: Optional[str] = optional_json_field("tagName", "non_null") @@ -59,10 +60,10 @@ def __post_init__(self): ): if value is not None: _nonblank(value, field_name) - if self.request_id is None: - self.request_id = uuid.uuid4().hex + if self.idempotency_key is None: + self.idempotency_key = uuid.uuid4().hex else: - _nonblank(self.request_id, "request_id") + _nonblank(self.idempotency_key, "idempotency_key") if self.snapshot_id is not None and ( type(self.snapshot_id) is not int or self.snapshot_id <= 0): raise ValueError("snapshot_id must be a positive integer") diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index f699c839fe2b..0b36885ae9d4 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -90,7 +90,11 @@ class PredictionTest(unittest.TestCase): def test_request_and_response_wire_format(self): request = SubmitPredictionRequest.from_spec(_spec()) body = json.loads(JSON.to_json(request)) - self.assertTrue(body["requestId"]) + self.assertTrue(body["idempotencyKey"]) + self.assertEqual( + body["idempotencyKey"], + json.loads(JSON.to_json(request))["idempotencyKey"], + ) self.assertEqual(["clip_id"], body["resultColumns"]) self.assertEqual({"result": "STRING"}, body["outputSchema"]) self.assertNotIn("snapshotId", body) @@ -212,7 +216,7 @@ def do_GET(self): "/v1/catalog/databases/a%2Fb/tables/clip+table/predictions", calls[-1][1], ) - self.assertTrue(calls[-1][2]["requestId"]) + self.assertTrue(calls[-1][2]["idempotencyKey"]) page = catalog.list_predictions( identifier, From e995032a0427d1f6e21b47f660d47f3f019c31be Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Mon, 16 Nov 2026 19:42:59 +0800 Subject: [PATCH 03/41] [core][python] Make prediction jobs catalog-scoped --- docs/docs/concepts/rest/rest-api.md | 1 + docs/static/rest-catalog-open-api.yaml | 59 ++++++++-------- .../org/apache/paimon/catalog/Prediction.java | 10 ++- .../paimon/catalog/PredictionSource.java | 68 +++++++++++++++++++ .../paimon/catalog/PredictionStatus.java | 2 +- .../java/org/apache/paimon/rest/RESTApi.java | 28 ++++---- .../org/apache/paimon/rest/ResourcePaths.java | 12 ++-- .../paimon/rest/auth/DLFOpenApiActions.java | 20 ++---- .../requests/SubmitPredictionRequest.java | 17 ++++- .../rest/responses/GetPredictionResponse.java | 13 +++- .../responses/ListPredictionsResponse.java | 2 +- .../paimon/rest/RESTApiPredictionTest.java | 11 ++- .../org/apache/paimon/catalog/Catalog.java | 32 +++------ .../paimon/catalog/DelegateCatalog.java | 17 ++--- .../org/apache/paimon/rest/RESTCatalog.java | 31 +++------ .../rest/RESTCatalogPredictionTest.java | 25 ++++--- .../pypaimon/api/auth/dlf_openapi_actions.py | 8 +-- paimon-python/pypaimon/api/prediction.py | 31 ++++++++- paimon-python/pypaimon/api/resource_paths.py | 16 ++--- paimon-python/pypaimon/api/rest_api.py | 27 +++----- paimon-python/pypaimon/catalog/catalog.py | 6 +- .../pypaimon/catalog/catalog_exception.py | 6 +- paimon-python/pypaimon/catalog/prediction.py | 7 +- .../pypaimon/catalog/rest/rest_catalog.py | 31 +++------ .../pypaimon/multimodal/prediction.py | 11 ++- paimon-python/pypaimon/multimodal/table.py | 13 ++-- .../pypaimon/tests/rest/prediction_test.py | 23 +++++-- .../tests/rest/rest_api_coverage_test.py | 3 +- 28 files changed, 307 insertions(+), 223 deletions(-) create mode 100644 paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSource.java diff --git a/docs/docs/concepts/rest/rest-api.md b/docs/docs/concepts/rest/rest-api.md index b3279b265231..286ba4059917 100644 --- a/docs/docs/concepts/rest/rest-api.md +++ b/docs/docs/concepts/rest/rest-api.md @@ -55,6 +55,7 @@ payloads, and error responses are defined in the OpenAPI specification. | Branches and tags | Manage named histories and retained snapshots. | Table-scoped `branches` and `tags`. | | Consumers | List and reset streaming consumer progress. | Table-scoped `consumers`. | | Views and functions | Manage reusable SQL and function definitions. | Database- and catalog-scoped `views` and `functions`. | +| Prediction jobs | Submit, list, inspect, and cancel asynchronous predictions. | Catalog-scoped `predictions`; the source table is part of the submission. | In this table, **table-scoped** means `/v1/{prefix}/databases/{database}/tables/{table}`. Catalog-wide listing and detail-listing diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 87f264768e87..c37feec54354 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -133,19 +133,17 @@ paths: "404": $ref: '#/components/responses/DatabaseOrModelNotExistErrorResponse' "501": {description: Models are not supported} - /v1/{prefix}/databases/{database}/tables/{table}/predictions: + /v1/{prefix}/predictions: post: tags: [prediction] - summary: Submit an asynchronous table prediction + summary: Submit a prediction job operationId: submitPrediction description: >- - Submits an asynchronous prediction with a fixed snapshot and model version. - Reusing an idempotencyKey for the same table and payload returns the existing - job; reusing it with a different payload returns 409. + Accepts a job after fixing the source snapshot and model version. Reusing an + idempotencyKey with the same payload returns the existing job; conflicting + reuse returns 409. parameters: - {name: prefix, in: path, required: true, schema: {type: string}} - - {name: database, in: path, required: true, schema: {type: string}} - - {name: table, in: path, required: true, schema: {type: string}} requestBody: required: true content: @@ -168,13 +166,13 @@ paths: "501": {description: Predictions are not supported} get: tags: [prediction] - summary: List table predictions + summary: List prediction jobs operationId: listPredictions description: Newest first. parameters: - {name: prefix, in: path, required: true, schema: {type: string}} - - {name: database, in: path, required: true, schema: {type: string}} - - {name: table, in: path, required: true, schema: {type: string}} + - {name: database, in: query, required: false, schema: {type: string}} + - {name: table, in: query, required: false, schema: {type: string}} - {name: name, in: query, required: false, schema: {type: string}} - name: status in: query @@ -185,23 +183,19 @@ paths: - {name: pageToken, in: query, required: false, schema: {type: string}} responses: "200": - description: Prediction metadata for the table + description: Prediction jobs content: application/json: schema: $ref: '#/components/schemas/ListPredictionsResponse' - "404": - $ref: '#/components/responses/TableNotExistErrorResponse' "501": {description: Predictions are not supported} - /v1/{prefix}/databases/{database}/tables/{table}/predictions/{jobId}: + /v1/{prefix}/predictions/{jobId}: get: tags: [prediction] - summary: Get a table prediction + summary: Get a prediction job operationId: getPrediction parameters: - {name: prefix, in: path, required: true, schema: {type: string}} - - {name: database, in: path, required: true, schema: {type: string}} - - {name: table, in: path, required: true, schema: {type: string}} - {name: jobId, in: path, required: true, schema: {type: string, minLength: 1}} responses: "200": @@ -211,20 +205,18 @@ paths: schema: $ref: '#/components/schemas/GetPredictionResponse' "404": - $ref: '#/components/responses/TableOrPredictionNotExistErrorResponse' + $ref: '#/components/responses/PredictionNotExistErrorResponse' "501": {description: Predictions are not supported} - /v1/{prefix}/databases/{database}/tables/{table}/predictions/{jobId}/cancel: + /v1/{prefix}/predictions/{jobId}/cancel: post: tags: [prediction] - summary: Cancel a table prediction + summary: Cancel a prediction job operationId: cancelPrediction description: >- Requests cancellation and returns the latest state. Terminal jobs are unchanged. parameters: - {name: prefix, in: path, required: true, schema: {type: string}} - - {name: database, in: path, required: true, schema: {type: string}} - - {name: table, in: path, required: true, schema: {type: string}} - {name: jobId, in: path, required: true, schema: {type: string, minLength: 1}} responses: "200": @@ -234,7 +226,7 @@ paths: schema: $ref: '#/components/schemas/GetPredictionResponse' "404": - $ref: '#/components/responses/TableOrPredictionNotExistErrorResponse' + $ref: '#/components/responses/PredictionNotExistErrorResponse' "501": {description: Predictions are not supported} /v1/config: get: @@ -2997,8 +2989,8 @@ components: resourceType: type: string enum: [ "DATABASE", "TABLE", "MODEL" ] - TableOrPredictionNotExistErrorResponse: - description: Database, table, or prediction not found + PredictionNotExistErrorResponse: + description: Prediction not found content: application/json: schema: @@ -3009,7 +3001,7 @@ components: properties: resourceType: type: string - enum: [ "DATABASE", "TABLE", "PREDICTION" ] + enum: [ "PREDICTION" ] TableNotExistErrorResponse: description: Not Found - TableNotExistException, the table does not exist @@ -4680,11 +4672,20 @@ components: PredictionStatus: type: string enum: [PENDING, RUNNING, SUCCEEDED, PARTIAL_SUCCESS, FAILED, CANCELLED] + PredictionSource: + type: object + additionalProperties: false + required: [database, table] + properties: + database: {type: string, minLength: 1} + table: {type: string, minLength: 1} SubmitPredictionRequest: type: object additionalProperties: false - required: [idempotencyKey, model, inputs, resultColumns, prompt, outputSchema] + required: [source, idempotencyKey, model, inputs, resultColumns, prompt, outputSchema] properties: + source: + $ref: '#/components/schemas/PredictionSource' name: type: string minLength: 1 @@ -4735,9 +4736,11 @@ components: GetPredictionResponse: type: object additionalProperties: false - required: [jobId, status, snapshotId, model, modelVersion, outputTable, createdAt] + required: [jobId, source, status, snapshotId, model, modelVersion, outputTable, createdAt] properties: jobId: {type: string, minLength: 1} + source: + $ref: '#/components/schemas/PredictionSource' name: {type: string, minLength: 1} status: $ref: '#/components/schemas/PredictionStatus' diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java index 4ec297abf071..398f8bf3a356 100644 --- a/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java @@ -24,11 +24,12 @@ import static org.apache.paimon.utils.Preconditions.checkArgument; -/** Durable metadata for one asynchronous table prediction. */ +/** Durable metadata for one asynchronous prediction job. */ @Experimental public class Prediction { private final String jobId; + private final PredictionSource source; private final @Nullable String name; private final PredictionStatus status; private final long snapshotId; @@ -45,6 +46,7 @@ public class Prediction { public Prediction( String jobId, + PredictionSource source, @Nullable String name, PredictionStatus status, long snapshotId, @@ -59,6 +61,7 @@ public Prediction( @Nullable Long succeeded, @Nullable Long failed) { checkNonBlank(jobId, "jobId"); + checkArgument(source != null, "source must not be null"); checkArgument(status != null, "status must not be null"); checkArgument(snapshotId > 0, "snapshotId must be positive"); checkNonBlank(model, "model"); @@ -71,6 +74,7 @@ public Prediction( checkOptionalNonNegative(succeeded, "succeeded"); checkOptionalNonNegative(failed, "failed"); this.jobId = jobId; + this.source = source; this.name = name; this.status = status; this.snapshotId = snapshotId; @@ -90,6 +94,10 @@ public String jobId() { return jobId; } + public PredictionSource source() { + return source; + } + @Nullable public String name() { return name; diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSource.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSource.java new file mode 100644 index 000000000000..4b3eeeea5e50 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSource.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.catalog; + +import org.apache.paimon.annotation.Experimental; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Source table of a prediction job. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class PredictionSource { + + private final String database; + private final String table; + + @JsonCreator + public PredictionSource( + @JsonProperty("database") String database, @JsonProperty("table") String table) { + checkNonBlank(database, "database"); + checkNonBlank(table, "table"); + this.database = database; + this.table = table; + } + + public PredictionSource(Identifier identifier) { + this(identifier.getDatabaseName(), identifier.getObjectName()); + } + + @JsonGetter("database") + public String database() { + return database; + } + + @JsonGetter("table") + public String table() { + return table; + } + + public Identifier toIdentifier() { + return Identifier.create(database, table); + } + + private static void checkNonBlank(String value, String name) { + checkArgument(value != null && !value.trim().isEmpty(), "%s must not be blank", name); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionStatus.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionStatus.java index 5f915d61c258..b6704008f9fd 100644 --- a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionStatus.java +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionStatus.java @@ -20,7 +20,7 @@ import org.apache.paimon.annotation.Experimental; -/** Lifecycle state of an asynchronous table prediction. */ +/** Lifecycle state of an asynchronous prediction job. */ @Experimental public enum PredictionStatus { PENDING, diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index 0297ff536f97..0f8b5799388f 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -1819,27 +1819,23 @@ public void dropModel(Identifier identifier) { } /** Submit an asynchronous prediction over a table. */ - public GetPredictionResponse submitPrediction( - Identifier identifier, SubmitPredictionRequest request) { + public GetPredictionResponse submitPrediction(SubmitPredictionRequest request) { return client.post( - resourcePaths.predictions(identifier.getDatabaseName(), identifier.getObjectName()), + resourcePaths.predictions(), request, GetPredictionResponse.class, restAuthFunction); } - /** Get one asynchronous table prediction by its unique job id. */ - public GetPredictionResponse getPrediction(Identifier identifier, String jobId) { + /** Get one asynchronous prediction by its unique job id. */ + public GetPredictionResponse getPrediction(String jobId) { return client.get( - resourcePaths.prediction( - identifier.getDatabaseName(), identifier.getObjectName(), jobId), - GetPredictionResponse.class, - restAuthFunction); + resourcePaths.prediction(jobId), GetPredictionResponse.class, restAuthFunction); } - /** List asynchronous predictions for a table, ordered newest first by the server. */ + /** List asynchronous predictions, ordered newest first by the server. */ public PagedList listPredictions( - Identifier identifier, + @Nullable Identifier source, @Nullable String name, @Nullable PredictionStatus status, @Nullable Integer maxResults, @@ -1848,12 +1844,13 @@ public PagedList listPredictions( buildPagedQueryParams( maxResults, pageToken, + Pair.of("database", source == null ? null : source.getDatabaseName()), + Pair.of("table", source == null ? null : source.getObjectName()), Pair.of(PREDICTION_NAME, name), Pair.of(PREDICTION_STATUS, status == null ? null : status.name())); ListPredictionsResponse response = client.get( - resourcePaths.predictions( - identifier.getDatabaseName(), identifier.getObjectName()), + resourcePaths.predictions(), queryParams, ListPredictionsResponse.class, restAuthFunction); @@ -1861,10 +1858,9 @@ public PagedList listPredictions( } /** Request cancellation and return the updated prediction metadata. */ - public GetPredictionResponse cancelPrediction(Identifier identifier, String jobId) { + public GetPredictionResponse cancelPrediction(String jobId) { return client.post( - resourcePaths.cancelPrediction( - identifier.getDatabaseName(), identifier.getObjectName(), jobId), + resourcePaths.cancelPrediction(jobId), null, GetPredictionResponse.class, restAuthFunction); diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java index da5975bd6ddd..4ee4f57eeb24 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -161,17 +161,17 @@ public String model(String databaseName, String modelName) { return SLASH.join(models(databaseName), encodeString(modelName)); } - public String predictions(String databaseName, String tableName) { - return SLASH.join(table(databaseName, tableName), PREDICTIONS); + public String predictions() { + return SLASH.join(V1, prefix, PREDICTIONS); } - public String prediction(String databaseName, String tableName, String jobId) { + public String prediction(String jobId) { checkArgument(jobId != null && !jobId.trim().isEmpty(), "jobId must not be blank"); - return SLASH.join(predictions(databaseName, tableName), encodeString(jobId)); + return SLASH.join(predictions(), encodeString(jobId)); } - public String cancelPrediction(String databaseName, String tableName, String jobId) { - return SLASH.join(prediction(databaseName, tableName, jobId), "cancel"); + public String cancelPrediction(String jobId) { + return SLASH.join(prediction(jobId), "cancel"); } private static void checkModelPathSegment(String value, String name) { diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java index 3cd08cfe7f83..1933d62a0778 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java @@ -51,22 +51,10 @@ final class DLFOpenApiActions { {"GET", "/v1/{prefix}/tables/id/{tableId}", "GetTableById"}, {"POST", "/v1/{prefix}/tables/rename", "RenameTable"}, {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/auth", "AuthTableQuery"}, - { - "POST", - "/v1/{prefix}/databases/{database}/tables/{table}/predictions", - "SubmitPrediction" - }, - {"GET", "/v1/{prefix}/databases/{database}/tables/{table}/predictions", "ListPredictions"}, - { - "GET", - "/v1/{prefix}/databases/{database}/tables/{table}/predictions/{jobId}", - "GetPrediction" - }, - { - "POST", - "/v1/{prefix}/databases/{database}/tables/{table}/predictions/{jobId}/cancel", - "CancelPrediction" - }, + {"POST", "/v1/{prefix}/predictions", "SubmitPrediction"}, + {"GET", "/v1/{prefix}/predictions", "ListPredictions"}, + {"GET", "/v1/{prefix}/predictions/{jobId}", "GetPrediction"}, + {"POST", "/v1/{prefix}/predictions/{jobId}/cancel", "CancelPrediction"}, {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/commit", "CommitTable"}, { "POST", diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java index c0d4915a63b9..3ee685966b15 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java @@ -18,6 +18,8 @@ package org.apache.paimon.rest.requests; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.PredictionSource; import org.apache.paimon.catalog.PredictionSpec; import org.apache.paimon.rest.RESTRequest; @@ -35,11 +37,12 @@ import static org.apache.paimon.utils.Preconditions.checkArgument; -/** Request for submitting an asynchronous table prediction. */ +/** Request for submitting an asynchronous prediction job. */ @JsonIgnoreProperties(ignoreUnknown = true) public class SubmitPredictionRequest implements RESTRequest { private static final String FIELD_NAME = "name"; + private static final String FIELD_SOURCE = "source"; private static final String FIELD_IDEMPOTENCY_KEY = "idempotencyKey"; private static final String FIELD_MODEL = "model"; private static final String FIELD_INPUTS = "inputs"; @@ -52,10 +55,12 @@ public class SubmitPredictionRequest implements RESTRequest { private static final String FIELD_OPTIONS = "options"; private final String idempotencyKey; + private final PredictionSource source; private final PredictionSpec spec; @JsonCreator public SubmitPredictionRequest( + @JsonProperty(FIELD_SOURCE) PredictionSource source, @Nullable @JsonProperty(FIELD_NAME) String name, @Nullable @JsonProperty(FIELD_IDEMPOTENCY_KEY) String idempotencyKey, @JsonProperty(FIELD_MODEL) String model, @@ -67,9 +72,11 @@ public SubmitPredictionRequest( @Nullable @JsonProperty(FIELD_SNAPSHOT_ID) Long snapshotId, @Nullable @JsonProperty(FIELD_TAG_NAME) String tagName, @Nullable @JsonProperty(FIELD_OPTIONS) Map options) { + checkArgument(source != null, "source must not be null"); checkArgument( idempotencyKey == null || !idempotencyKey.trim().isEmpty(), "idempotencyKey must not be blank"); + this.source = source; this.idempotencyKey = idempotencyKey == null ? UUID.randomUUID().toString() : idempotencyKey; this.spec = @@ -86,8 +93,9 @@ public SubmitPredictionRequest( options); } - public SubmitPredictionRequest(PredictionSpec spec) { + public SubmitPredictionRequest(Identifier source, PredictionSpec spec) { this( + new PredictionSource(source), spec.name(), null, spec.model(), @@ -101,6 +109,11 @@ public SubmitPredictionRequest(PredictionSpec spec) { spec.options()); } + @JsonGetter(FIELD_SOURCE) + public PredictionSource getSource() { + return source; + } + @Nullable @JsonInclude(JsonInclude.Include.NON_NULL) @JsonGetter(FIELD_NAME) diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java index c6da27abccc9..bae5a61ba2b6 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java @@ -19,6 +19,7 @@ package org.apache.paimon.rest.responses; import org.apache.paimon.catalog.Prediction; +import org.apache.paimon.catalog.PredictionSource; import org.apache.paimon.catalog.PredictionStatus; import org.apache.paimon.rest.RESTResponse; @@ -30,11 +31,12 @@ import javax.annotation.Nullable; -/** Response containing durable metadata for one asynchronous table prediction. */ +/** Response containing durable metadata for one asynchronous prediction job. */ @JsonIgnoreProperties(ignoreUnknown = true) public class GetPredictionResponse implements RESTResponse { private final String jobId; + private final PredictionSource source; private final @Nullable String name; private final PredictionStatus status; private final long snapshotId; @@ -52,6 +54,7 @@ public class GetPredictionResponse implements RESTResponse { @JsonCreator public GetPredictionResponse( @JsonProperty("jobId") String jobId, + @JsonProperty("source") PredictionSource source, @Nullable @JsonProperty("name") String name, @JsonProperty("status") PredictionStatus status, @JsonProperty("snapshotId") long snapshotId, @@ -66,6 +69,7 @@ public GetPredictionResponse( @Nullable @JsonProperty("succeeded") Long succeeded, @Nullable @JsonProperty("failed") Long failed) { this.jobId = jobId; + this.source = source; this.name = name; this.status = status; this.snapshotId = snapshotId; @@ -84,6 +88,7 @@ public GetPredictionResponse( public GetPredictionResponse(Prediction prediction) { this( prediction.jobId(), + prediction.source(), prediction.name(), prediction.status(), prediction.snapshotId(), @@ -104,6 +109,11 @@ public String getJobId() { return jobId; } + @JsonGetter("source") + public PredictionSource getSource() { + return source; + } + @Nullable @JsonInclude(JsonInclude.Include.NON_NULL) @JsonGetter("name") @@ -186,6 +196,7 @@ public Long getFailed() { public Prediction toPrediction() { return new Prediction( jobId, + source, name, status, snapshotId, diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListPredictionsResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListPredictionsResponse.java index a45cecfba285..9a78780f988f 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListPredictionsResponse.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListPredictionsResponse.java @@ -29,7 +29,7 @@ import java.util.Collections; import java.util.List; -/** Response for listing asynchronous table predictions. */ +/** Response for listing asynchronous prediction jobs. */ @JsonIgnoreProperties(ignoreUnknown = true) public class ListPredictionsResponse implements PagedResponse { diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java index 8a1663aa8601..c309d2e167e2 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java @@ -18,7 +18,9 @@ package org.apache.paimon.rest; +import org.apache.paimon.catalog.Identifier; import org.apache.paimon.catalog.Prediction; +import org.apache.paimon.catalog.PredictionSource; import org.apache.paimon.catalog.PredictionSpec; import org.apache.paimon.catalog.PredictionStatus; import org.apache.paimon.rest.requests.SubmitPredictionRequest; @@ -35,12 +37,13 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -/** Wire-format tests for asynchronous table prediction metadata. */ +/** Wire-format tests for asynchronous prediction metadata. */ class RESTApiPredictionTest { @Test void testRequestAndResponseWireFormat() throws Exception { - SubmitPredictionRequest request = new SubmitPredictionRequest(spec()); + SubmitPredictionRequest request = + new SubmitPredictionRequest(Identifier.create("quality", "clips"), spec()); Map requestMap = RESTApi.fromJson(RESTApi.toJson(request), Map.class); assertThat(requestMap.get("idempotencyKey")).isInstanceOf(String.class); assertThat((String) requestMap.get("idempotencyKey")).isNotBlank(); @@ -50,6 +53,9 @@ void testRequestAndResponseWireFormat() throws Exception { .containsEntry("model", "quality.qwen_vl") .doesNotContainKeys("name", "where", "snapshotId", "tagName"); assertThat(requestMap.get("resultColumns")).isEqualTo(Collections.singletonList("clip_id")); + assertThat((Map) requestMap.get("source")) + .containsEntry("database", "quality") + .containsEntry("table", "clips"); GetPredictionResponse response = new GetPredictionResponse(prediction()); GetPredictionResponse decoded = @@ -128,6 +134,7 @@ private static PredictionSpec spec() { private static Prediction prediction() { return new Prediction( "job-1", + new PredictionSource("quality", "clips"), null, PredictionStatus.PENDING, 7, diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java index d2c684d25ff4..ccc999f2f10e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java @@ -1205,26 +1205,23 @@ default Prediction submitPrediction(Identifier identifier, PredictionSpec spec) throw new UnsupportedOperationException("Current Catalog does not support predictions"); } - /** Get one asynchronous table prediction by its unique job id. */ - default Prediction getPrediction(Identifier identifier, String jobId) - throws TableNotExistException, PredictionNotExistException { + /** Get one asynchronous prediction by its unique job id. */ + default Prediction getPrediction(String jobId) throws PredictionNotExistException { throw new UnsupportedOperationException("Current Catalog does not support predictions"); } - /** List asynchronous predictions for a table. */ + /** List asynchronous predictions, optionally filtered by source table. */ default PagedList listPredictions( - Identifier identifier, + @Nullable Identifier source, @Nullable String name, @Nullable PredictionStatus status, @Nullable Integer maxResults, - @Nullable String pageToken) - throws TableNotExistException { + @Nullable String pageToken) { throw new UnsupportedOperationException("Current Catalog does not support predictions"); } /** Request cancellation and return the updated prediction metadata. */ - default Prediction cancelPrediction(Identifier identifier, String jobId) - throws TableNotExistException, PredictionNotExistException { + default Prediction cancelPrediction(String jobId) throws PredictionNotExistException { throw new UnsupportedOperationException("Current Catalog does not support predictions"); } @@ -1971,26 +1968,19 @@ public Identifier identifier() { /** Exception for trying to access a prediction that doesn't exist. */ class PredictionNotExistException extends Exception { - private static final String MSG = "Prediction %s for table %s doesn't exist."; + private static final String MSG = "Prediction %s doesn't exist."; - private final Identifier identifier; private final String jobId; - public PredictionNotExistException(Identifier identifier, String jobId) { - this(identifier, jobId, null); + public PredictionNotExistException(String jobId) { + this(jobId, null); } - public PredictionNotExistException( - Identifier identifier, String jobId, @Nullable Throwable cause) { - super(String.format(MSG, jobId, identifier.getFullName()), cause); - this.identifier = identifier; + public PredictionNotExistException(String jobId, @Nullable Throwable cause) { + super(String.format(MSG, jobId), cause); this.jobId = jobId; } - public Identifier identifier() { - return identifier; - } - public String jobId() { return jobId; } diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java index 08579b26fd88..df6a9dc40ce9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java @@ -414,26 +414,23 @@ public Prediction submitPrediction(Identifier identifier, PredictionSpec spec) } @Override - public Prediction getPrediction(Identifier identifier, String jobId) - throws TableNotExistException, PredictionNotExistException { - return wrapped.getPrediction(identifier, jobId); + public Prediction getPrediction(String jobId) throws PredictionNotExistException { + return wrapped.getPrediction(jobId); } @Override public PagedList listPredictions( - Identifier identifier, + @Nullable Identifier source, @Nullable String name, @Nullable PredictionStatus status, @Nullable Integer maxResults, - @Nullable String pageToken) - throws TableNotExistException { - return wrapped.listPredictions(identifier, name, status, maxResults, pageToken); + @Nullable String pageToken) { + return wrapped.listPredictions(source, name, status, maxResults, pageToken); } @Override - public Prediction cancelPrediction(Identifier identifier, String jobId) - throws TableNotExistException, PredictionNotExistException { - return wrapped.cancelPrediction(identifier, jobId); + public Prediction cancelPrediction(String jobId) throws PredictionNotExistException { + return wrapped.cancelPrediction(jobId); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index a19de2e013bb..6e6ed887e9eb 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -1218,7 +1218,7 @@ public void dropModel(Identifier identifier, boolean ignoreIfNotExists) public Prediction submitPrediction(Identifier identifier, PredictionSpec spec) throws TableNotExistException, ModelNotExistException { try { - return api.submitPrediction(identifier, new SubmitPredictionRequest(spec)) + return api.submitPrediction(new SubmitPredictionRequest(identifier, spec)) .toPrediction(); } catch (NoSuchResourceException e) { if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_MODEL)) { @@ -1231,15 +1231,11 @@ public Prediction submitPrediction(Identifier identifier, PredictionSpec spec) } @Override - public Prediction getPrediction(Identifier identifier, String jobId) - throws TableNotExistException, PredictionNotExistException { + public Prediction getPrediction(String jobId) throws PredictionNotExistException { try { - return api.getPrediction(identifier, jobId).toPrediction(); + return api.getPrediction(jobId).toPrediction(); } catch (NoSuchResourceException e) { - if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_PREDICTION)) { - throw new PredictionNotExistException(identifier, jobId, e); - } - throw new TableNotExistException(identifier, e); + throw new PredictionNotExistException(jobId, e); } catch (NotImplementedException e) { throw new UnsupportedOperationException("REST Catalog does not support predictions", e); } @@ -1247,15 +1243,14 @@ public Prediction getPrediction(Identifier identifier, String jobId) @Override public PagedList listPredictions( - Identifier identifier, + @Nullable Identifier source, @Nullable String name, @Nullable PredictionStatus status, @Nullable Integer maxResults, - @Nullable String pageToken) - throws TableNotExistException { + @Nullable String pageToken) { try { PagedList response = - api.listPredictions(identifier, name, status, maxResults, pageToken); + api.listPredictions(source, name, status, maxResults, pageToken); return new PagedList<>( response.getElements().stream() .map( @@ -1263,23 +1258,17 @@ public PagedList listPredictions( ::toPrediction) .collect(Collectors.toList()), response.getNextPageToken()); - } catch (NoSuchResourceException e) { - throw new TableNotExistException(identifier, e); } catch (NotImplementedException e) { throw new UnsupportedOperationException("REST Catalog does not support predictions", e); } } @Override - public Prediction cancelPrediction(Identifier identifier, String jobId) - throws TableNotExistException, PredictionNotExistException { + public Prediction cancelPrediction(String jobId) throws PredictionNotExistException { try { - return api.cancelPrediction(identifier, jobId).toPrediction(); + return api.cancelPrediction(jobId).toPrediction(); } catch (NoSuchResourceException e) { - if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_PREDICTION)) { - throw new PredictionNotExistException(identifier, jobId, e); - } - throw new TableNotExistException(identifier, e); + throw new PredictionNotExistException(jobId, e); } catch (NotImplementedException e) { throw new UnsupportedOperationException("REST Catalog does not support predictions", e); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java index c6ced782abee..8c31650b5e40 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java @@ -39,7 +39,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -/** REST catalog tests for durable asynchronous table predictions. */ +/** REST catalog tests for durable asynchronous predictions. */ class RESTCatalogPredictionTest { @Test @@ -53,11 +53,13 @@ void testPredictionLifecycleUsesSharedRestContract() throws Exception { Prediction submitted = catalog.submitPrediction(table, spec()); assertThat(submitted.jobId()).isEqualTo("job-1"); RecordedRequest submit = server.takeRequest(10, TimeUnit.SECONDS); - assertThat(submit.getPath()) - .isEqualTo("/v1/catalog/databases/quality/tables/clips/predictions"); + assertThat(submit.getPath()).isEqualTo("/v1/catalog/predictions"); Map request = RESTApi.fromJson(submit.getBody().readUtf8(), Map.class); assertThat(request.get("idempotencyKey")).isInstanceOf(String.class); assertThat((String) request.get("idempotencyKey")).isNotBlank(); + assertThat((Map) request.get("source")) + .containsEntry("database", "quality") + .containsEntry("table", "clips"); server.enqueue( json( @@ -72,18 +74,18 @@ void testPredictionLifecycleUsesSharedRestContract() throws Exception { assertThat(page.getNextPageToken()).isEqualTo("next"); String listPath = server.takeRequest(10, TimeUnit.SECONDS).getPath(); assertThat(listPath) + .contains("database=quality") + .contains("table=clips") .contains("name=daily-quality") .contains("status=RUNNING") .contains("maxResults=10") .contains("pageToken=page-1"); server.enqueue(json(predictionJson("CANCELLED"), 200)); - assertThat(catalog.cancelPrediction(table, "job/1").status()) + assertThat(catalog.cancelPrediction("job/1").status()) .isEqualTo(PredictionStatus.CANCELLED); assertThat(server.takeRequest(10, TimeUnit.SECONDS).getPath()) - .isEqualTo( - "/v1/catalog/databases/quality/tables/clips/predictions/" - + "job%2F1/cancel"); + .isEqualTo("/v1/catalog/predictions/job%2F1/cancel"); } } @@ -98,12 +100,8 @@ void testPredictionNotFoundPreservesResourceType() throws Exception { assertThatThrownBy(() -> catalog.submitPrediction(table, spec())) .isInstanceOf(Catalog.ModelNotExistException.class); - server.enqueue(notFound("TABLE", "clips")); - assertThatThrownBy(() -> catalog.getPrediction(table, "job-1")) - .isInstanceOf(Catalog.TableNotExistException.class); - server.enqueue(notFound("PREDICTION", "job-1")); - assertThatThrownBy(() -> catalog.getPrediction(table, "job-1")) + assertThatThrownBy(() -> catalog.getPrediction("job-1")) .isInstanceOf(Catalog.PredictionNotExistException.class); } } @@ -132,7 +130,8 @@ private static Catalog catalog(MockWebServer server) { } private static String predictionJson(String status) { - return "{\"jobId\":\"job-1\",\"name\":\"daily-quality\",\"status\":\"" + return "{\"jobId\":\"job-1\",\"source\":{\"database\":\"quality\"," + + "\"table\":\"clips\"},\"name\":\"daily-quality\",\"status\":\"" + status + "\",\"snapshotId\":7,\"model\":\"quality.qwen_vl\"," + "\"modelVersion\":\"1\",\"outputTable\":\"system.results\"," diff --git a/paimon-python/pypaimon/api/auth/dlf_openapi_actions.py b/paimon-python/pypaimon/api/auth/dlf_openapi_actions.py index fc1b99d64ef1..27dc3c80f879 100644 --- a/paimon-python/pypaimon/api/auth/dlf_openapi_actions.py +++ b/paimon-python/pypaimon/api/auth/dlf_openapi_actions.py @@ -102,10 +102,10 @@ ("POST", "/v1/{prefix}/databases/{database}/models", "CreateModel"), ("GET", "/v1/{prefix}/databases/{database}/models/{model}", "GetModel"), ("DELETE", "/v1/{prefix}/databases/{database}/models/{model}", "DropModel"), - ("POST", "/v1/{prefix}/databases/{database}/tables/{table}/predictions", "SubmitPrediction"), - ("GET", "/v1/{prefix}/databases/{database}/tables/{table}/predictions", "ListPredictions"), - ("GET", "/v1/{prefix}/databases/{database}/tables/{table}/predictions/{jobId}", "GetPrediction"), - ("POST", "/v1/{prefix}/databases/{database}/tables/{table}/predictions/{jobId}/cancel", "CancelPrediction"), + ("POST", "/v1/{prefix}/predictions", "SubmitPrediction"), + ("GET", "/v1/{prefix}/predictions", "ListPredictions"), + ("GET", "/v1/{prefix}/predictions/{jobId}", "GetPrediction"), + ("POST", "/v1/{prefix}/predictions/{jobId}/cancel", "CancelPrediction"), ("GET", "/v1/{prefix}/permissions", "ListPermissionAssignments"), ("POST", "/v1/{prefix}/permissions/grant", "GrantPermissionAssignment"), ("POST", "/v1/{prefix}/permissions/revoke", "RevokePermissionAssignment"), diff --git a/paimon-python/pypaimon/api/prediction.py b/paimon-python/pypaimon/api/prediction.py index 40aa09158a2f..f590cc666d08 100644 --- a/paimon-python/pypaimon/api/prediction.py +++ b/paimon-python/pypaimon/api/prediction.py @@ -30,10 +30,29 @@ _nonblank, ) from pypaimon.common.json_util import json_field, optional_json_field +from pypaimon.common.identifier import Identifier + + +@dataclass +class PredictionSource: + database: str + table: str + + def __post_init__(self): + _nonblank(self.database, "database") + _nonblank(self.table, "table") + + @classmethod + def from_identifier(cls, identifier: Identifier) -> "PredictionSource": + return cls(identifier.get_database_name(), identifier.get_object_name()) + + def to_identifier(self) -> Identifier: + return Identifier.create(self.database, self.table) @dataclass class SubmitPredictionRequest(RESTRequest): + source: PredictionSource model: str inputs: Dict[str, str] result_columns: List[str] = json_field("resultColumns") @@ -79,8 +98,11 @@ def __post_init__(self): self.options = dict(self.options) @classmethod - def from_spec(cls, spec: PredictionSpec) -> "SubmitPredictionRequest": + def from_spec( + cls, source: Identifier, spec: PredictionSpec + ) -> "SubmitPredictionRequest": return cls( + source=PredictionSource.from_identifier(source), name=spec.name, model=spec.model, inputs=spec.inputs, @@ -104,6 +126,7 @@ class CancelPredictionRequest(RESTRequest): @dataclass class GetPredictionResponse(RESTResponse): job_id: str = json_field("jobId") + source: PredictionSource = None status: str = "" snapshot_id: int = json_field("snapshotId", default=0) model: str = "" @@ -122,6 +145,11 @@ class GetPredictionResponse(RESTResponse): def from_dict(cls, data: Dict[str, Any]) -> "GetPredictionResponse": return cls( job_id=data.get("jobId"), + source=( + PredictionSource(**data.get("source")) + if isinstance(data.get("source"), dict) + else data.get("source") + ), name=data.get("name"), status=data.get("status"), snapshot_id=data.get("snapshotId"), @@ -140,6 +168,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "GetPredictionResponse": def to_prediction(self) -> Prediction: return Prediction( job_id=self.job_id, + source=self.source.to_identifier(), name=self.name, status=PredictionStatus(self.status), snapshot_id=self.snapshot_id, diff --git a/paimon-python/pypaimon/api/resource_paths.py b/paimon-python/pypaimon/api/resource_paths.py index c7f37d852bad..9a98bb46bb34 100644 --- a/paimon-python/pypaimon/api/resource_paths.py +++ b/paimon-python/pypaimon/api/resource_paths.py @@ -133,20 +133,16 @@ def model(self, database_name: str, model_name: str) -> str: self.models(database_name), RESTUtil.encode_string(model_name) ) - def predictions(self, database_name: str, table_name: str) -> str: - return "{}/{}".format( - self.table(database_name, table_name), self.PREDICTIONS) + def predictions(self) -> str: + return "{}/{}".format(self.base_path, self.PREDICTIONS) - def prediction(self, database_name: str, table_name: str, job_id: str) -> str: + def prediction(self, job_id: str) -> str: if not isinstance(job_id, str) or not job_id.strip(): raise ValueError("job_id must be a non-empty string") - return "{}/{}".format( - self.predictions(database_name, table_name), RESTUtil.encode_string(job_id)) + return "{}/{}".format(self.predictions(), RESTUtil.encode_string(job_id)) - def cancel_prediction( - self, database_name: str, table_name: str, job_id: str) -> str: - return "{}/cancel".format( - self.prediction(database_name, table_name, job_id)) + def cancel_prediction(self, job_id: str) -> str: + return "{}/cancel".format(self.prediction(job_id)) def tags(self, database_name: str, table_name: str) -> str: return "{}/{}/{}/{}/{}/{}".format( diff --git a/paimon-python/pypaimon/api/rest_api.py b/paimon-python/pypaimon/api/rest_api.py index 0cfbdbc71ef8..434b20fae334 100755 --- a/paimon-python/pypaimon/api/rest_api.py +++ b/paimon-python/pypaimon/api/rest_api.py @@ -792,37 +792,35 @@ def drop_model(self, identifier: Identifier) -> None: def submit_prediction( self, - identifier: Identifier, request: SubmitPredictionRequest, ) -> GetPredictionResponse: - database_name, table_name = self._validate_resource_path(identifier) return self.client.post_with_response_type( - self.resource_paths.predictions(database_name, table_name), + self.resource_paths.predictions(), request, GetPredictionResponse, self.rest_auth_function, ) - def get_prediction( - self, identifier: Identifier, job_id: str - ) -> GetPredictionResponse: - database_name, table_name = self._validate_resource_path(identifier) + def get_prediction(self, job_id: str) -> GetPredictionResponse: return self.client.get( - self.resource_paths.prediction(database_name, table_name, job_id), + self.resource_paths.prediction(job_id), GetPredictionResponse, self.rest_auth_function, ) def list_predictions( self, - identifier: Identifier, + source: Optional[Identifier] = None, name: Optional[str] = None, status: Optional[PredictionStatus] = None, max_results: Optional[int] = None, page_token: Optional[str] = None, ) -> PagedList[GetPredictionResponse]: - database_name, table_name = self._validate_resource_path(identifier) query_params = {} + if source is not None: + database_name, table_name = self._validate_resource_path(source) + self._put_query_parameter(query_params, "database", database_name) + self._put_query_parameter(query_params, "table", table_name) self._put_query_parameter(query_params, "name", name) self._put_query_parameter( query_params, "status", status.value if status is not None else None) @@ -830,19 +828,16 @@ def list_predictions( query_params[self.MAX_RESULTS] = str(max_results) self._put_query_parameter(query_params, self.PAGE_TOKEN, page_token) response = self.client.get_with_params( - self.resource_paths.predictions(database_name, table_name), + self.resource_paths.predictions(), query_params, ListPredictionsResponse, self.rest_auth_function, ) return PagedList(response.predictions, response.next_page_token) - def cancel_prediction( - self, identifier: Identifier, job_id: str - ) -> GetPredictionResponse: - database_name, table_name = self._validate_resource_path(identifier) + def cancel_prediction(self, job_id: str) -> GetPredictionResponse: return self.client.post_with_response_type( - self.resource_paths.cancel_prediction(database_name, table_name, job_id), + self.resource_paths.cancel_prediction(job_id), CancelPredictionRequest(), GetPredictionResponse, self.rest_auth_function, diff --git a/paimon-python/pypaimon/catalog/catalog.py b/paimon-python/pypaimon/catalog/catalog.py index 62436dee2f22..3655376f09dc 100644 --- a/paimon-python/pypaimon/catalog/catalog.py +++ b/paimon-python/pypaimon/catalog/catalog.py @@ -122,7 +122,6 @@ def submit_prediction( def get_prediction( self, - identifier: Union[str, Identifier], job_id: str, ) -> 'Prediction': """Get one prediction by its unique job id.""" @@ -131,19 +130,18 @@ def get_prediction( def list_predictions( self, - identifier: Union[str, Identifier], + source: Optional[Union[str, Identifier]] = None, name: Optional[str] = None, status: Optional['PredictionStatus'] = None, max_results: Optional[int] = None, page_token: Optional[str] = None, ) -> PagedList['Prediction']: - """List predictions for a table, optionally filtered by name or status.""" + """List predictions, optionally filtered by source table.""" raise NotImplementedError( "list_predictions is not supported by this catalog.") def cancel_prediction( self, - identifier: Union[str, Identifier], job_id: str, ) -> 'Prediction': """Request cancellation and return the updated prediction.""" diff --git a/paimon-python/pypaimon/catalog/catalog_exception.py b/paimon-python/pypaimon/catalog/catalog_exception.py index b0d104b4b9ec..a50c6ef66142 100644 --- a/paimon-python/pypaimon/catalog/catalog_exception.py +++ b/paimon-python/pypaimon/catalog/catalog_exception.py @@ -138,11 +138,9 @@ def __init__(self, identifier: Identifier): class PredictionNotExistException(CatalogException): """Prediction not exist exception""" - def __init__(self, identifier: Identifier, job_id: str): - self.identifier = identifier + def __init__(self, job_id: str): self.job_id = job_id - super().__init__( - f"Prediction {job_id} for table {identifier.get_full_name()} does not exist") + super().__init__(f"Prediction {job_id} does not exist") class ColumnNotExistException(CatalogException): diff --git a/paimon-python/pypaimon/catalog/prediction.py b/paimon-python/pypaimon/catalog/prediction.py index 83aa00915551..f3d65cf41077 100644 --- a/paimon-python/pypaimon/catalog/prediction.py +++ b/paimon-python/pypaimon/catalog/prediction.py @@ -19,6 +19,8 @@ from enum import Enum from typing import Any, Dict, List, Optional +from pypaimon.common.identifier import Identifier + class PredictionStatus(str, Enum): PENDING = "PENDING" @@ -80,9 +82,10 @@ def __post_init__(self): @dataclass class Prediction: - """Durable metadata for one asynchronous table prediction.""" + """Durable metadata for one asynchronous prediction job.""" job_id: str + source: Identifier status: PredictionStatus snapshot_id: int model: str @@ -99,6 +102,8 @@ class Prediction: def __post_init__(self): _nonblank(self.job_id, "job_id") + if not isinstance(self.source, Identifier): + raise ValueError("source must be an Identifier") if isinstance(self.status, str): self.status = PredictionStatus(self.status) if not isinstance(self.status, PredictionStatus): diff --git a/paimon-python/pypaimon/catalog/rest/rest_catalog.py b/paimon-python/pypaimon/catalog/rest/rest_catalog.py index e453bdcca9f0..c39ac92377e7 100644 --- a/paimon-python/pypaimon/catalog/rest/rest_catalog.py +++ b/paimon-python/pypaimon/catalog/rest/rest_catalog.py @@ -692,7 +692,7 @@ def submit_prediction( check_not_system_table(identifier, "submit_prediction") try: return self.rest_api.submit_prediction( - identifier, SubmitPredictionRequest.from_spec(spec)).to_prediction() + SubmitPredictionRequest.from_spec(identifier, spec)).to_prediction() except NoSuchResourceException as e: if e.resource_type == ErrorResponse.RESOURCE_TYPE_MODEL: raise ModelNotExistException(Identifier.from_string(spec.model)) from e @@ -703,58 +703,45 @@ def submit_prediction( def get_prediction( self, - identifier: Union[str, Identifier], job_id: str, ) -> Prediction: - if not isinstance(identifier, Identifier): - identifier = Identifier.from_string(identifier) try: - return self.rest_api.get_prediction(identifier, job_id).to_prediction() + return self.rest_api.get_prediction(job_id).to_prediction() except NoSuchResourceException as e: - if e.resource_type == ErrorResponse.RESOURCE_TYPE_PREDICTION: - raise PredictionNotExistException(identifier, job_id) from e - raise TableNotExistException(identifier) from e + raise PredictionNotExistException(job_id) from e except NotImplementedException as e: raise NotImplementedError( "REST Catalog does not support predictions") from e def list_predictions( self, - identifier: Union[str, Identifier], + source: Optional[Union[str, Identifier]] = None, name: Optional[str] = None, status: Optional[PredictionStatus] = None, max_results: Optional[int] = None, page_token: Optional[str] = None, ) -> PagedList[Prediction]: - if not isinstance(identifier, Identifier): - identifier = Identifier.from_string(identifier) + if source is not None and not isinstance(source, Identifier): + source = Identifier.from_string(source) try: page = self.rest_api.list_predictions( - identifier, name, status, max_results, page_token) + source, name, status, max_results, page_token) return PagedList( [value.to_prediction() for value in page.elements], page.next_page_token, ) - except NoSuchResourceException as e: - raise TableNotExistException(identifier) from e except NotImplementedException as e: raise NotImplementedError( "REST Catalog does not support predictions") from e def cancel_prediction( self, - identifier: Union[str, Identifier], job_id: str, ) -> Prediction: - if not isinstance(identifier, Identifier): - identifier = Identifier.from_string(identifier) try: - return self.rest_api.cancel_prediction( - identifier, job_id).to_prediction() + return self.rest_api.cancel_prediction(job_id).to_prediction() except NoSuchResourceException as e: - if e.resource_type == ErrorResponse.RESOURCE_TYPE_PREDICTION: - raise PredictionNotExistException(identifier, job_id) from e - raise TableNotExistException(identifier) from e + raise PredictionNotExistException(job_id) from e except NotImplementedException as e: raise NotImplementedError( "REST Catalog does not support predictions") from e diff --git a/paimon-python/pypaimon/multimodal/prediction.py b/paimon-python/pypaimon/multimodal/prediction.py index 6a82e2b300fa..99c7d238e802 100644 --- a/paimon-python/pypaimon/multimodal/prediction.py +++ b/paimon-python/pypaimon/multimodal/prediction.py @@ -20,16 +20,13 @@ from typing import Optional from pypaimon.catalog.prediction import Prediction -from pypaimon.common.identifier import Identifier class PredictionJob: - """Client handle for a durable asynchronous table prediction.""" + """Client handle for a durable asynchronous prediction job.""" - def __init__( - self, catalog, identifier: Identifier, prediction: Prediction): + def __init__(self, catalog, prediction: Prediction): self._catalog = catalog - self._identifier = identifier self._prediction = prediction @property @@ -41,7 +38,7 @@ def __getattr__(self, name): def refresh(self) -> 'PredictionJob': self._prediction = self._catalog.get_prediction( - self._identifier, self._prediction.job_id) + self._prediction.job_id) return self def wait( @@ -77,5 +74,5 @@ def wait( def cancel(self) -> 'PredictionJob': self._prediction = self._catalog.cancel_prediction( - self._identifier, self._prediction.job_id) + self._prediction.job_id) return self diff --git a/paimon-python/pypaimon/multimodal/table.py b/paimon-python/pypaimon/multimodal/table.py index 5523c84152cc..89fa358f7090 100644 --- a/paimon-python/pypaimon/multimodal/table.py +++ b/paimon-python/pypaimon/multimodal/table.py @@ -154,17 +154,14 @@ def predict( ) prediction = self.catalog.submit_prediction( self.raw_table.identifier, spec) - return PredictionJob( - self.catalog, self.raw_table.identifier, prediction) + return PredictionJob(self.catalog, prediction) def get_prediction(self, job_id): """Return a client handle for one asynchronous prediction.""" from pypaimon.multimodal.prediction import PredictionJob - prediction = self.catalog.get_prediction( - self.raw_table.identifier, job_id) - return PredictionJob( - self.catalog, self.raw_table.identifier, prediction) + prediction = self.catalog.get_prediction(job_id) + return PredictionJob(self.catalog, prediction) def list_predictions( self, @@ -181,7 +178,7 @@ def list_predictions( if isinstance(status, str): status = PredictionStatus(status) page = self.catalog.list_predictions( - self.raw_table.identifier, + source=self.raw_table.identifier, name=name, status=status, max_results=max_results, @@ -189,7 +186,7 @@ def list_predictions( ) return PagedList( [ - PredictionJob(self.catalog, self.raw_table.identifier, value) + PredictionJob(self.catalog, value) for value in page.elements ], page.next_page_token, diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index 0b36885ae9d4..95513d501f89 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -25,6 +25,7 @@ from pypaimon.api.prediction import ( GetPredictionResponse, ListPredictionsResponse, + PredictionSource, SubmitPredictionRequest, ) from pypaimon.api.rest_api import RESTApi @@ -58,6 +59,7 @@ def _spec(**changes): def _prediction(status=PredictionStatus.PENDING, **changes): values = dict( job_id="job-1", + source=Identifier.create("quality", "clips"), name="daily-quality", status=status, snapshot_id=7, @@ -73,6 +75,7 @@ def _prediction(status=PredictionStatus.PENDING, **changes): def _response(status="PENDING", **changes): values = dict( job_id="job-1", + source=PredictionSource("quality", "clips"), name="daily-quality", status=status, snapshot_id=7, @@ -88,7 +91,8 @@ def _response(status="PENDING", **changes): class PredictionTest(unittest.TestCase): def test_request_and_response_wire_format(self): - request = SubmitPredictionRequest.from_spec(_spec()) + request = SubmitPredictionRequest.from_spec( + Identifier.create("quality", "clips"), _spec()) body = json.loads(JSON.to_json(request)) self.assertTrue(body["idempotencyKey"]) self.assertEqual( @@ -97,6 +101,8 @@ def test_request_and_response_wire_format(self): ) self.assertEqual(["clip_id"], body["resultColumns"]) self.assertEqual({"result": "STRING"}, body["outputSchema"]) + self.assertEqual( + {"database": "quality", "table": "clips"}, body["source"]) self.assertNotIn("snapshotId", body) response = JSON.from_json( @@ -157,8 +163,7 @@ def test_wait_is_client_side_polling(self): _prediction( PredictionStatus.SUCCEEDED, total=2, succeeded=2, failed=0), ] - job = PredictionJob( - catalog, Identifier.create("quality", "clips"), _prediction()) + job = PredictionJob(catalog, _prediction()) result = job.wait(timeout=timedelta(seconds=1), poll_interval=0) self.assertIs(job, result) self.assertEqual(PredictionStatus.SUCCEEDED, job.status) @@ -213,9 +218,13 @@ def do_GET(self): self.assertEqual("job-1", catalog.submit_prediction( identifier, _spec()).job_id) self.assertEqual( - "/v1/catalog/databases/a%2Fb/tables/clip+table/predictions", + "/v1/catalog/predictions", calls[-1][1], ) + self.assertEqual( + {"database": "a/b", "table": "clip table"}, + calls[-1][2]["source"], + ) self.assertTrue(calls[-1][2]["idempotencyKey"]) page = catalog.list_predictions( @@ -227,13 +236,15 @@ def do_GET(self): ) self.assertEqual("job-1", page.elements[0].job_id) self.assertIn("name=daily-quality", calls[-1][1]) + self.assertIn("database=a%2Fb", calls[-1][1]) + self.assertIn("table=clip+table", calls[-1][1]) self.assertIn("status=RUNNING", calls[-1][1]) self.assertIn("maxResults=10", calls[-1][1]) self.assertIn("pageToken=p1", calls[-1][1]) - catalog.get_prediction(identifier, "job/1") + catalog.get_prediction("job/1") self.assertTrue(calls[-1][1].endswith("/predictions/job%2F1")) - catalog.cancel_prediction(identifier, "job/1") + catalog.cancel_prediction("job/1") self.assertTrue( calls[-1][1].endswith("/predictions/job%2F1/cancel")) finally: diff --git a/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py b/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py index a20920797c2b..bfa1999c14f6 100644 --- a/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py +++ b/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py @@ -27,7 +27,7 @@ from pypaimon.api.rest_api import RESTApi from pypaimon.api.model import CreateModelRequest -from pypaimon.api.prediction import SubmitPredictionRequest +from pypaimon.api.prediction import PredictionSource, SubmitPredictionRequest from pypaimon.catalog.model import ModelField, ProviderModelDefinition from pypaimon.common.identifier import Identifier from pypaimon.function.function import FunctionImpl @@ -65,6 +65,7 @@ ), ), SubmitPredictionRequest: SubmitPredictionRequest( + source=PredictionSource("db", "t"), model="db.qwen_vl", inputs={"video": "content"}, result_columns=["id"], From df909e14136607dbdc86a15424c5aeaf13bd0655 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Mon, 16 Nov 2026 19:54:13 +0800 Subject: [PATCH 04/41] [core][python] Align prediction identifiers --- docs/static/rest-catalog-open-api.yaml | 12 +++++----- .../org/apache/paimon/catalog/Prediction.java | 12 +++++----- .../java/org/apache/paimon/rest/RESTApi.java | 12 ++++++---- .../org/apache/paimon/rest/ResourcePaths.java | 12 ++++++---- .../paimon/rest/auth/DLFOpenApiActions.java | 4 ++-- .../rest/responses/GetPredictionResponse.java | 16 ++++++------- .../paimon/rest/RESTApiPredictionTest.java | 2 +- .../org/apache/paimon/catalog/Catalog.java | 22 ++++++++--------- .../paimon/catalog/DelegateCatalog.java | 8 +++---- .../org/apache/paimon/rest/RESTCatalog.java | 12 +++++----- .../rest/RESTCatalogPredictionTest.java | 16 +++++++------ .../pypaimon/api/auth/dlf_openapi_actions.py | 4 ++-- paimon-python/pypaimon/api/prediction.py | 6 ++--- paimon-python/pypaimon/api/resource_paths.py | 13 +++++----- paimon-python/pypaimon/api/rest_api.py | 8 +++---- paimon-python/pypaimon/catalog/catalog.py | 6 ++--- .../pypaimon/catalog/catalog_exception.py | 6 ++--- paimon-python/pypaimon/catalog/prediction.py | 4 ++-- .../pypaimon/catalog/rest/rest_catalog.py | 12 +++++----- .../pypaimon/multimodal/prediction.py | 6 ++--- paimon-python/pypaimon/multimodal/table.py | 4 ++-- .../pypaimon/tests/rest/prediction_test.py | 24 ++++++++++--------- 22 files changed, 115 insertions(+), 106 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index c37feec54354..5d90c2ffcff8 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -189,14 +189,14 @@ paths: schema: $ref: '#/components/schemas/ListPredictionsResponse' "501": {description: Predictions are not supported} - /v1/{prefix}/predictions/{jobId}: + /v1/{prefix}/predictions/{predictionId}: get: tags: [prediction] summary: Get a prediction job operationId: getPrediction parameters: - {name: prefix, in: path, required: true, schema: {type: string}} - - {name: jobId, in: path, required: true, schema: {type: string, minLength: 1}} + - {name: predictionId, in: path, required: true, schema: {type: string, minLength: 1}} responses: "200": description: Prediction metadata @@ -207,7 +207,7 @@ paths: "404": $ref: '#/components/responses/PredictionNotExistErrorResponse' "501": {description: Predictions are not supported} - /v1/{prefix}/predictions/{jobId}/cancel: + /v1/{prefix}/predictions/{predictionId}/cancel: post: tags: [prediction] summary: Cancel a prediction job @@ -217,7 +217,7 @@ paths: unchanged. parameters: - {name: prefix, in: path, required: true, schema: {type: string}} - - {name: jobId, in: path, required: true, schema: {type: string, minLength: 1}} + - {name: predictionId, in: path, required: true, schema: {type: string, minLength: 1}} responses: "200": description: Latest prediction metadata @@ -4736,9 +4736,9 @@ components: GetPredictionResponse: type: object additionalProperties: false - required: [jobId, source, status, snapshotId, model, modelVersion, outputTable, createdAt] + required: [predictionId, source, status, snapshotId, model, modelVersion, outputTable, createdAt] properties: - jobId: {type: string, minLength: 1} + predictionId: {type: string, minLength: 1} source: $ref: '#/components/schemas/PredictionSource' name: {type: string, minLength: 1} diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java index 398f8bf3a356..366103aac38c 100644 --- a/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java @@ -28,7 +28,7 @@ @Experimental public class Prediction { - private final String jobId; + private final String predictionId; private final PredictionSource source; private final @Nullable String name; private final PredictionStatus status; @@ -45,7 +45,7 @@ public class Prediction { private final @Nullable Long failed; public Prediction( - String jobId, + String predictionId, PredictionSource source, @Nullable String name, PredictionStatus status, @@ -60,7 +60,7 @@ public Prediction( @Nullable Long total, @Nullable Long succeeded, @Nullable Long failed) { - checkNonBlank(jobId, "jobId"); + checkNonBlank(predictionId, "predictionId"); checkArgument(source != null, "source must not be null"); checkArgument(status != null, "status must not be null"); checkArgument(snapshotId > 0, "snapshotId must be positive"); @@ -73,7 +73,7 @@ public Prediction( checkOptionalNonNegative(total, "total"); checkOptionalNonNegative(succeeded, "succeeded"); checkOptionalNonNegative(failed, "failed"); - this.jobId = jobId; + this.predictionId = predictionId; this.source = source; this.name = name; this.status = status; @@ -90,8 +90,8 @@ public Prediction( this.failed = failed; } - public String jobId() { - return jobId; + public String predictionId() { + return predictionId; } public PredictionSource source() { diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index 0f8b5799388f..faf0258e22e8 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -1827,10 +1827,12 @@ public GetPredictionResponse submitPrediction(SubmitPredictionRequest request) { restAuthFunction); } - /** Get one asynchronous prediction by its unique job id. */ - public GetPredictionResponse getPrediction(String jobId) { + /** Get one asynchronous prediction by its unique id. */ + public GetPredictionResponse getPrediction(String predictionId) { return client.get( - resourcePaths.prediction(jobId), GetPredictionResponse.class, restAuthFunction); + resourcePaths.prediction(predictionId), + GetPredictionResponse.class, + restAuthFunction); } /** List asynchronous predictions, ordered newest first by the server. */ @@ -1858,9 +1860,9 @@ public PagedList listPredictions( } /** Request cancellation and return the updated prediction metadata. */ - public GetPredictionResponse cancelPrediction(String jobId) { + public GetPredictionResponse cancelPrediction(String predictionId) { return client.post( - resourcePaths.cancelPrediction(jobId), + resourcePaths.cancelPrediction(predictionId), null, GetPredictionResponse.class, restAuthFunction); diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java index 4ee4f57eeb24..120d8eaafb91 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -165,13 +165,15 @@ public String predictions() { return SLASH.join(V1, prefix, PREDICTIONS); } - public String prediction(String jobId) { - checkArgument(jobId != null && !jobId.trim().isEmpty(), "jobId must not be blank"); - return SLASH.join(predictions(), encodeString(jobId)); + public String prediction(String predictionId) { + checkArgument( + predictionId != null && !predictionId.trim().isEmpty(), + "predictionId must not be blank"); + return SLASH.join(predictions(), encodeString(predictionId)); } - public String cancelPrediction(String jobId) { - return SLASH.join(prediction(jobId), "cancel"); + public String cancelPrediction(String predictionId) { + return SLASH.join(prediction(predictionId), "cancel"); } private static void checkModelPathSegment(String value, String name) { diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java index 1933d62a0778..727443d8d61e 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java @@ -53,8 +53,8 @@ final class DLFOpenApiActions { {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/auth", "AuthTableQuery"}, {"POST", "/v1/{prefix}/predictions", "SubmitPrediction"}, {"GET", "/v1/{prefix}/predictions", "ListPredictions"}, - {"GET", "/v1/{prefix}/predictions/{jobId}", "GetPrediction"}, - {"POST", "/v1/{prefix}/predictions/{jobId}/cancel", "CancelPrediction"}, + {"GET", "/v1/{prefix}/predictions/{predictionId}", "GetPrediction"}, + {"POST", "/v1/{prefix}/predictions/{predictionId}/cancel", "CancelPrediction"}, {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/commit", "CommitTable"}, { "POST", diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java index bae5a61ba2b6..f036b40e9c5d 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java @@ -35,7 +35,7 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class GetPredictionResponse implements RESTResponse { - private final String jobId; + private final String predictionId; private final PredictionSource source; private final @Nullable String name; private final PredictionStatus status; @@ -53,7 +53,7 @@ public class GetPredictionResponse implements RESTResponse { @JsonCreator public GetPredictionResponse( - @JsonProperty("jobId") String jobId, + @JsonProperty("predictionId") String predictionId, @JsonProperty("source") PredictionSource source, @Nullable @JsonProperty("name") String name, @JsonProperty("status") PredictionStatus status, @@ -68,7 +68,7 @@ public GetPredictionResponse( @Nullable @JsonProperty("total") Long total, @Nullable @JsonProperty("succeeded") Long succeeded, @Nullable @JsonProperty("failed") Long failed) { - this.jobId = jobId; + this.predictionId = predictionId; this.source = source; this.name = name; this.status = status; @@ -87,7 +87,7 @@ public GetPredictionResponse( public GetPredictionResponse(Prediction prediction) { this( - prediction.jobId(), + prediction.predictionId(), prediction.source(), prediction.name(), prediction.status(), @@ -104,9 +104,9 @@ public GetPredictionResponse(Prediction prediction) { prediction.failed()); } - @JsonGetter("jobId") - public String getJobId() { - return jobId; + @JsonGetter("predictionId") + public String getPredictionId() { + return predictionId; } @JsonGetter("source") @@ -195,7 +195,7 @@ public Long getFailed() { public Prediction toPrediction() { return new Prediction( - jobId, + predictionId, source, name, status, diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java index c309d2e167e2..684b96e07786 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java @@ -133,7 +133,7 @@ private static PredictionSpec spec() { private static Prediction prediction() { return new Prediction( - "job-1", + "prediction-1", new PredictionSource("quality", "clips"), null, PredictionStatus.PENDING, diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java index ccc999f2f10e..b3e7f836dd7e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java @@ -1205,8 +1205,8 @@ default Prediction submitPrediction(Identifier identifier, PredictionSpec spec) throw new UnsupportedOperationException("Current Catalog does not support predictions"); } - /** Get one asynchronous prediction by its unique job id. */ - default Prediction getPrediction(String jobId) throws PredictionNotExistException { + /** Get one asynchronous prediction by its unique id. */ + default Prediction getPrediction(String predictionId) throws PredictionNotExistException { throw new UnsupportedOperationException("Current Catalog does not support predictions"); } @@ -1221,7 +1221,7 @@ default PagedList listPredictions( } /** Request cancellation and return the updated prediction metadata. */ - default Prediction cancelPrediction(String jobId) throws PredictionNotExistException { + default Prediction cancelPrediction(String predictionId) throws PredictionNotExistException { throw new UnsupportedOperationException("Current Catalog does not support predictions"); } @@ -1970,19 +1970,19 @@ class PredictionNotExistException extends Exception { private static final String MSG = "Prediction %s doesn't exist."; - private final String jobId; + private final String predictionId; - public PredictionNotExistException(String jobId) { - this(jobId, null); + public PredictionNotExistException(String predictionId) { + this(predictionId, null); } - public PredictionNotExistException(String jobId, @Nullable Throwable cause) { - super(String.format(MSG, jobId), cause); - this.jobId = jobId; + public PredictionNotExistException(String predictionId, @Nullable Throwable cause) { + super(String.format(MSG, predictionId), cause); + this.predictionId = predictionId; } - public String jobId() { - return jobId; + public String predictionId() { + return predictionId; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java index df6a9dc40ce9..b4183cc30552 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java @@ -414,8 +414,8 @@ public Prediction submitPrediction(Identifier identifier, PredictionSpec spec) } @Override - public Prediction getPrediction(String jobId) throws PredictionNotExistException { - return wrapped.getPrediction(jobId); + public Prediction getPrediction(String predictionId) throws PredictionNotExistException { + return wrapped.getPrediction(predictionId); } @Override @@ -429,8 +429,8 @@ public PagedList listPredictions( } @Override - public Prediction cancelPrediction(String jobId) throws PredictionNotExistException { - return wrapped.cancelPrediction(jobId); + public Prediction cancelPrediction(String predictionId) throws PredictionNotExistException { + return wrapped.cancelPrediction(predictionId); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 6e6ed887e9eb..0bec83191dac 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -1231,11 +1231,11 @@ public Prediction submitPrediction(Identifier identifier, PredictionSpec spec) } @Override - public Prediction getPrediction(String jobId) throws PredictionNotExistException { + public Prediction getPrediction(String predictionId) throws PredictionNotExistException { try { - return api.getPrediction(jobId).toPrediction(); + return api.getPrediction(predictionId).toPrediction(); } catch (NoSuchResourceException e) { - throw new PredictionNotExistException(jobId, e); + throw new PredictionNotExistException(predictionId, e); } catch (NotImplementedException e) { throw new UnsupportedOperationException("REST Catalog does not support predictions", e); } @@ -1264,11 +1264,11 @@ public PagedList listPredictions( } @Override - public Prediction cancelPrediction(String jobId) throws PredictionNotExistException { + public Prediction cancelPrediction(String predictionId) throws PredictionNotExistException { try { - return api.cancelPrediction(jobId).toPrediction(); + return api.cancelPrediction(predictionId).toPrediction(); } catch (NoSuchResourceException e) { - throw new PredictionNotExistException(jobId, e); + throw new PredictionNotExistException(predictionId, e); } catch (NotImplementedException e) { throw new UnsupportedOperationException("REST Catalog does not support predictions", e); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java index 8c31650b5e40..efab562c56d1 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java @@ -51,7 +51,7 @@ void testPredictionLifecycleUsesSharedRestContract() throws Exception { server.enqueue(json(predictionJson("PENDING"), 202)); Prediction submitted = catalog.submitPrediction(table, spec()); - assertThat(submitted.jobId()).isEqualTo("job-1"); + assertThat(submitted.predictionId()).isEqualTo("prediction-1"); RecordedRequest submit = server.takeRequest(10, TimeUnit.SECONDS); assertThat(submit.getPath()).isEqualTo("/v1/catalog/predictions"); Map request = RESTApi.fromJson(submit.getBody().readUtf8(), Map.class); @@ -70,7 +70,9 @@ void testPredictionLifecycleUsesSharedRestContract() throws Exception { PagedList page = catalog.listPredictions( table, "daily-quality", PredictionStatus.RUNNING, 10, "page-1"); - assertThat(page.getElements()).extracting(Prediction::jobId).containsExactly("job-1"); + assertThat(page.getElements()) + .extracting(Prediction::predictionId) + .containsExactly("prediction-1"); assertThat(page.getNextPageToken()).isEqualTo("next"); String listPath = server.takeRequest(10, TimeUnit.SECONDS).getPath(); assertThat(listPath) @@ -82,10 +84,10 @@ void testPredictionLifecycleUsesSharedRestContract() throws Exception { .contains("pageToken=page-1"); server.enqueue(json(predictionJson("CANCELLED"), 200)); - assertThat(catalog.cancelPrediction("job/1").status()) + assertThat(catalog.cancelPrediction("prediction/1").status()) .isEqualTo(PredictionStatus.CANCELLED); assertThat(server.takeRequest(10, TimeUnit.SECONDS).getPath()) - .isEqualTo("/v1/catalog/predictions/job%2F1/cancel"); + .isEqualTo("/v1/catalog/predictions/prediction%2F1/cancel"); } } @@ -100,8 +102,8 @@ void testPredictionNotFoundPreservesResourceType() throws Exception { assertThatThrownBy(() -> catalog.submitPrediction(table, spec())) .isInstanceOf(Catalog.ModelNotExistException.class); - server.enqueue(notFound("PREDICTION", "job-1")); - assertThatThrownBy(() -> catalog.getPrediction("job-1")) + server.enqueue(notFound("PREDICTION", "prediction-1")); + assertThatThrownBy(() -> catalog.getPrediction("prediction-1")) .isInstanceOf(Catalog.PredictionNotExistException.class); } } @@ -130,7 +132,7 @@ private static Catalog catalog(MockWebServer server) { } private static String predictionJson(String status) { - return "{\"jobId\":\"job-1\",\"source\":{\"database\":\"quality\"," + return "{\"predictionId\":\"prediction-1\",\"source\":{\"database\":\"quality\"," + "\"table\":\"clips\"},\"name\":\"daily-quality\",\"status\":\"" + status + "\",\"snapshotId\":7,\"model\":\"quality.qwen_vl\"," diff --git a/paimon-python/pypaimon/api/auth/dlf_openapi_actions.py b/paimon-python/pypaimon/api/auth/dlf_openapi_actions.py index 27dc3c80f879..86d8437bbb04 100644 --- a/paimon-python/pypaimon/api/auth/dlf_openapi_actions.py +++ b/paimon-python/pypaimon/api/auth/dlf_openapi_actions.py @@ -104,8 +104,8 @@ ("DELETE", "/v1/{prefix}/databases/{database}/models/{model}", "DropModel"), ("POST", "/v1/{prefix}/predictions", "SubmitPrediction"), ("GET", "/v1/{prefix}/predictions", "ListPredictions"), - ("GET", "/v1/{prefix}/predictions/{jobId}", "GetPrediction"), - ("POST", "/v1/{prefix}/predictions/{jobId}/cancel", "CancelPrediction"), + ("GET", "/v1/{prefix}/predictions/{predictionId}", "GetPrediction"), + ("POST", "/v1/{prefix}/predictions/{predictionId}/cancel", "CancelPrediction"), ("GET", "/v1/{prefix}/permissions", "ListPermissionAssignments"), ("POST", "/v1/{prefix}/permissions/grant", "GrantPermissionAssignment"), ("POST", "/v1/{prefix}/permissions/revoke", "RevokePermissionAssignment"), diff --git a/paimon-python/pypaimon/api/prediction.py b/paimon-python/pypaimon/api/prediction.py index f590cc666d08..69263238f109 100644 --- a/paimon-python/pypaimon/api/prediction.py +++ b/paimon-python/pypaimon/api/prediction.py @@ -125,7 +125,7 @@ class CancelPredictionRequest(RESTRequest): @dataclass class GetPredictionResponse(RESTResponse): - job_id: str = json_field("jobId") + prediction_id: str = json_field("predictionId") source: PredictionSource = None status: str = "" snapshot_id: int = json_field("snapshotId", default=0) @@ -144,7 +144,7 @@ class GetPredictionResponse(RESTResponse): @classmethod def from_dict(cls, data: Dict[str, Any]) -> "GetPredictionResponse": return cls( - job_id=data.get("jobId"), + prediction_id=data.get("predictionId"), source=( PredictionSource(**data.get("source")) if isinstance(data.get("source"), dict) @@ -167,7 +167,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "GetPredictionResponse": def to_prediction(self) -> Prediction: return Prediction( - job_id=self.job_id, + prediction_id=self.prediction_id, source=self.source.to_identifier(), name=self.name, status=PredictionStatus(self.status), diff --git a/paimon-python/pypaimon/api/resource_paths.py b/paimon-python/pypaimon/api/resource_paths.py index 9a98bb46bb34..81cbc772d0c2 100644 --- a/paimon-python/pypaimon/api/resource_paths.py +++ b/paimon-python/pypaimon/api/resource_paths.py @@ -136,13 +136,14 @@ def model(self, database_name: str, model_name: str) -> str: def predictions(self) -> str: return "{}/{}".format(self.base_path, self.PREDICTIONS) - def prediction(self, job_id: str) -> str: - if not isinstance(job_id, str) or not job_id.strip(): - raise ValueError("job_id must be a non-empty string") - return "{}/{}".format(self.predictions(), RESTUtil.encode_string(job_id)) + def prediction(self, prediction_id: str) -> str: + if not isinstance(prediction_id, str) or not prediction_id.strip(): + raise ValueError("prediction_id must be a non-empty string") + return "{}/{}".format( + self.predictions(), RESTUtil.encode_string(prediction_id)) - def cancel_prediction(self, job_id: str) -> str: - return "{}/cancel".format(self.prediction(job_id)) + def cancel_prediction(self, prediction_id: str) -> str: + return "{}/cancel".format(self.prediction(prediction_id)) def tags(self, database_name: str, table_name: str) -> str: return "{}/{}/{}/{}/{}/{}".format( diff --git a/paimon-python/pypaimon/api/rest_api.py b/paimon-python/pypaimon/api/rest_api.py index 434b20fae334..f68c1856575e 100755 --- a/paimon-python/pypaimon/api/rest_api.py +++ b/paimon-python/pypaimon/api/rest_api.py @@ -801,9 +801,9 @@ def submit_prediction( self.rest_auth_function, ) - def get_prediction(self, job_id: str) -> GetPredictionResponse: + def get_prediction(self, prediction_id: str) -> GetPredictionResponse: return self.client.get( - self.resource_paths.prediction(job_id), + self.resource_paths.prediction(prediction_id), GetPredictionResponse, self.rest_auth_function, ) @@ -835,9 +835,9 @@ def list_predictions( ) return PagedList(response.predictions, response.next_page_token) - def cancel_prediction(self, job_id: str) -> GetPredictionResponse: + def cancel_prediction(self, prediction_id: str) -> GetPredictionResponse: return self.client.post_with_response_type( - self.resource_paths.cancel_prediction(job_id), + self.resource_paths.cancel_prediction(prediction_id), CancelPredictionRequest(), GetPredictionResponse, self.rest_auth_function, diff --git a/paimon-python/pypaimon/catalog/catalog.py b/paimon-python/pypaimon/catalog/catalog.py index 3655376f09dc..dbe670113e67 100644 --- a/paimon-python/pypaimon/catalog/catalog.py +++ b/paimon-python/pypaimon/catalog/catalog.py @@ -122,9 +122,9 @@ def submit_prediction( def get_prediction( self, - job_id: str, + prediction_id: str, ) -> 'Prediction': - """Get one prediction by its unique job id.""" + """Get one prediction by its unique id.""" raise NotImplementedError( "get_prediction is not supported by this catalog.") @@ -142,7 +142,7 @@ def list_predictions( def cancel_prediction( self, - job_id: str, + prediction_id: str, ) -> 'Prediction': """Request cancellation and return the updated prediction.""" raise NotImplementedError( diff --git a/paimon-python/pypaimon/catalog/catalog_exception.py b/paimon-python/pypaimon/catalog/catalog_exception.py index a50c6ef66142..934e720d6dae 100644 --- a/paimon-python/pypaimon/catalog/catalog_exception.py +++ b/paimon-python/pypaimon/catalog/catalog_exception.py @@ -138,9 +138,9 @@ def __init__(self, identifier: Identifier): class PredictionNotExistException(CatalogException): """Prediction not exist exception""" - def __init__(self, job_id: str): - self.job_id = job_id - super().__init__(f"Prediction {job_id} does not exist") + def __init__(self, prediction_id: str): + self.prediction_id = prediction_id + super().__init__(f"Prediction {prediction_id} does not exist") class ColumnNotExistException(CatalogException): diff --git a/paimon-python/pypaimon/catalog/prediction.py b/paimon-python/pypaimon/catalog/prediction.py index f3d65cf41077..d9131bbe8c9e 100644 --- a/paimon-python/pypaimon/catalog/prediction.py +++ b/paimon-python/pypaimon/catalog/prediction.py @@ -84,7 +84,7 @@ def __post_init__(self): class Prediction: """Durable metadata for one asynchronous prediction job.""" - job_id: str + prediction_id: str source: Identifier status: PredictionStatus snapshot_id: int @@ -101,7 +101,7 @@ class Prediction: failed: Optional[int] = None def __post_init__(self): - _nonblank(self.job_id, "job_id") + _nonblank(self.prediction_id, "prediction_id") if not isinstance(self.source, Identifier): raise ValueError("source must be an Identifier") if isinstance(self.status, str): diff --git a/paimon-python/pypaimon/catalog/rest/rest_catalog.py b/paimon-python/pypaimon/catalog/rest/rest_catalog.py index c39ac92377e7..a30bbdfd112f 100644 --- a/paimon-python/pypaimon/catalog/rest/rest_catalog.py +++ b/paimon-python/pypaimon/catalog/rest/rest_catalog.py @@ -703,12 +703,12 @@ def submit_prediction( def get_prediction( self, - job_id: str, + prediction_id: str, ) -> Prediction: try: - return self.rest_api.get_prediction(job_id).to_prediction() + return self.rest_api.get_prediction(prediction_id).to_prediction() except NoSuchResourceException as e: - raise PredictionNotExistException(job_id) from e + raise PredictionNotExistException(prediction_id) from e except NotImplementedException as e: raise NotImplementedError( "REST Catalog does not support predictions") from e @@ -736,12 +736,12 @@ def list_predictions( def cancel_prediction( self, - job_id: str, + prediction_id: str, ) -> Prediction: try: - return self.rest_api.cancel_prediction(job_id).to_prediction() + return self.rest_api.cancel_prediction(prediction_id).to_prediction() except NoSuchResourceException as e: - raise PredictionNotExistException(job_id) from e + raise PredictionNotExistException(prediction_id) from e except NotImplementedException as e: raise NotImplementedError( "REST Catalog does not support predictions") from e diff --git a/paimon-python/pypaimon/multimodal/prediction.py b/paimon-python/pypaimon/multimodal/prediction.py index 99c7d238e802..fe1364203c58 100644 --- a/paimon-python/pypaimon/multimodal/prediction.py +++ b/paimon-python/pypaimon/multimodal/prediction.py @@ -38,7 +38,7 @@ def __getattr__(self, name): def refresh(self) -> 'PredictionJob': self._prediction = self._catalog.get_prediction( - self._prediction.job_id) + self._prediction.prediction_id) return self def wait( @@ -65,7 +65,7 @@ def wait( if remaining <= 0: raise TimeoutError( "Prediction {} did not finish within {}".format( - self._prediction.job_id, timeout)) + self._prediction.prediction_id, timeout)) time.sleep(min(poll_interval, remaining)) else: time.sleep(poll_interval) @@ -74,5 +74,5 @@ def wait( def cancel(self) -> 'PredictionJob': self._prediction = self._catalog.cancel_prediction( - self._prediction.job_id) + self._prediction.prediction_id) return self diff --git a/paimon-python/pypaimon/multimodal/table.py b/paimon-python/pypaimon/multimodal/table.py index 89fa358f7090..d429630f5b71 100644 --- a/paimon-python/pypaimon/multimodal/table.py +++ b/paimon-python/pypaimon/multimodal/table.py @@ -156,11 +156,11 @@ def predict( self.raw_table.identifier, spec) return PredictionJob(self.catalog, prediction) - def get_prediction(self, job_id): + def get_prediction(self, prediction_id): """Return a client handle for one asynchronous prediction.""" from pypaimon.multimodal.prediction import PredictionJob - prediction = self.catalog.get_prediction(job_id) + prediction = self.catalog.get_prediction(prediction_id) return PredictionJob(self.catalog, prediction) def list_predictions( diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index 95513d501f89..3d907c28bfc8 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -58,7 +58,7 @@ def _spec(**changes): def _prediction(status=PredictionStatus.PENDING, **changes): values = dict( - job_id="job-1", + prediction_id="prediction-1", source=Identifier.create("quality", "clips"), name="daily-quality", status=status, @@ -74,7 +74,7 @@ def _prediction(status=PredictionStatus.PENDING, **changes): def _response(status="PENDING", **changes): values = dict( - job_id="job-1", + prediction_id="prediction-1", source=PredictionSource("quality", "clips"), name="daily-quality", status=status, @@ -113,7 +113,7 @@ def test_request_and_response_wire_format(self): JSON.to_json(ListPredictionsResponse([_response()], "next")), ListPredictionsResponse, ) - self.assertEqual("job-1", page.predictions[0].job_id) + self.assertEqual("prediction-1", page.predictions[0].prediction_id) self.assertEqual("next", page.next_page_token) def test_validation(self): @@ -150,7 +150,7 @@ def test_table_submit_fixes_view_and_returns_job(self): ) self.assertIsInstance(job, PredictionJob) - self.assertEqual("job-1", job.job_id) + self.assertEqual("prediction-1", job.prediction_id) sent = catalog.submit_prediction.call_args.args[1] self.assertEqual(7, sent.snapshot_id) self.assertIsNone(sent.tag_name) @@ -215,8 +215,8 @@ def do_GET(self): catalog.rest_api = api identifier = Identifier.create("a/b", "clip table") - self.assertEqual("job-1", catalog.submit_prediction( - identifier, _spec()).job_id) + self.assertEqual("prediction-1", catalog.submit_prediction( + identifier, _spec()).prediction_id) self.assertEqual( "/v1/catalog/predictions", calls[-1][1], @@ -234,7 +234,7 @@ def do_GET(self): max_results=10, page_token="p1", ) - self.assertEqual("job-1", page.elements[0].job_id) + self.assertEqual("prediction-1", page.elements[0].prediction_id) self.assertIn("name=daily-quality", calls[-1][1]) self.assertIn("database=a%2Fb", calls[-1][1]) self.assertIn("table=clip+table", calls[-1][1]) @@ -242,11 +242,13 @@ def do_GET(self): self.assertIn("maxResults=10", calls[-1][1]) self.assertIn("pageToken=p1", calls[-1][1]) - catalog.get_prediction("job/1") - self.assertTrue(calls[-1][1].endswith("/predictions/job%2F1")) - catalog.cancel_prediction("job/1") + catalog.get_prediction("prediction/1") self.assertTrue( - calls[-1][1].endswith("/predictions/job%2F1/cancel")) + calls[-1][1].endswith("/predictions/prediction%2F1")) + catalog.cancel_prediction("prediction/1") + self.assertTrue( + calls[-1][1].endswith( + "/predictions/prediction%2F1/cancel")) finally: server.shutdown() server.server_close() From e5cc6e3c21bee553756e5554bd411f4caa7574e0 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Mon, 16 Nov 2026 20:25:09 +0800 Subject: [PATCH 05/41] [core][python] Simplify prediction lifecycle metadata --- docs/static/rest-catalog-open-api.yaml | 3 -- .../org/apache/paimon/catalog/Prediction.java | 29 +------------ .../rest/responses/GetPredictionResponse.java | 42 ++----------------- .../paimon/rest/RESTApiPredictionTest.java | 3 -- paimon-python/pypaimon/api/prediction.py | 9 ---- paimon-python/pypaimon/catalog/prediction.py | 6 +-- .../pypaimon/tests/rest/prediction_test.py | 3 +- 7 files changed, 6 insertions(+), 89 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 5d90c2ffcff8..dcae938d2b44 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -4752,9 +4752,6 @@ components: startedAt: {type: integer, format: int64, minimum: 0} finishedAt: {type: integer, format: int64, minimum: 0} error: {type: string} - total: {type: integer, format: int64, minimum: 0} - succeeded: {type: integer, format: int64, minimum: 0} - failed: {type: integer, format: int64, minimum: 0} ListPredictionsResponse: type: object required: [predictions] diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java index 366103aac38c..a0b4c41bd8d7 100644 --- a/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java @@ -40,9 +40,6 @@ public class Prediction { private final @Nullable Long startedAt; private final @Nullable Long finishedAt; private final @Nullable String error; - private final @Nullable Long total; - private final @Nullable Long succeeded; - private final @Nullable Long failed; public Prediction( String predictionId, @@ -56,10 +53,7 @@ public Prediction( long createdAt, @Nullable Long startedAt, @Nullable Long finishedAt, - @Nullable String error, - @Nullable Long total, - @Nullable Long succeeded, - @Nullable Long failed) { + @Nullable String error) { checkNonBlank(predictionId, "predictionId"); checkArgument(source != null, "source must not be null"); checkArgument(status != null, "status must not be null"); @@ -70,9 +64,6 @@ public Prediction( checkArgument(createdAt >= 0, "createdAt must not be negative"); checkOptionalNonNegative(startedAt, "startedAt"); checkOptionalNonNegative(finishedAt, "finishedAt"); - checkOptionalNonNegative(total, "total"); - checkOptionalNonNegative(succeeded, "succeeded"); - checkOptionalNonNegative(failed, "failed"); this.predictionId = predictionId; this.source = source; this.name = name; @@ -85,9 +76,6 @@ public Prediction( this.startedAt = startedAt; this.finishedAt = finishedAt; this.error = error; - this.total = total; - this.succeeded = succeeded; - this.failed = failed; } public String predictionId() { @@ -142,21 +130,6 @@ public String error() { return error; } - @Nullable - public Long total() { - return total; - } - - @Nullable - public Long succeeded() { - return succeeded; - } - - @Nullable - public Long failed() { - return failed; - } - private static void checkNonBlank(String value, String name) { checkArgument(value != null && !value.trim().isEmpty(), "%s must not be blank", name); } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java index f036b40e9c5d..642110b087c1 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java @@ -47,9 +47,6 @@ public class GetPredictionResponse implements RESTResponse { private final @Nullable Long startedAt; private final @Nullable Long finishedAt; private final @Nullable String error; - private final @Nullable Long total; - private final @Nullable Long succeeded; - private final @Nullable Long failed; @JsonCreator public GetPredictionResponse( @@ -64,10 +61,7 @@ public GetPredictionResponse( @JsonProperty("createdAt") long createdAt, @Nullable @JsonProperty("startedAt") Long startedAt, @Nullable @JsonProperty("finishedAt") Long finishedAt, - @Nullable @JsonProperty("error") String error, - @Nullable @JsonProperty("total") Long total, - @Nullable @JsonProperty("succeeded") Long succeeded, - @Nullable @JsonProperty("failed") Long failed) { + @Nullable @JsonProperty("error") String error) { this.predictionId = predictionId; this.source = source; this.name = name; @@ -80,9 +74,6 @@ public GetPredictionResponse( this.startedAt = startedAt; this.finishedAt = finishedAt; this.error = error; - this.total = total; - this.succeeded = succeeded; - this.failed = failed; } public GetPredictionResponse(Prediction prediction) { @@ -98,10 +89,7 @@ public GetPredictionResponse(Prediction prediction) { prediction.createdAt(), prediction.startedAt(), prediction.finishedAt(), - prediction.error(), - prediction.total(), - prediction.succeeded(), - prediction.failed()); + prediction.error()); } @JsonGetter("predictionId") @@ -172,27 +160,6 @@ public String getError() { return error; } - @Nullable - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonGetter("total") - public Long getTotal() { - return total; - } - - @Nullable - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonGetter("succeeded") - public Long getSucceeded() { - return succeeded; - } - - @Nullable - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonGetter("failed") - public Long getFailed() { - return failed; - } - public Prediction toPrediction() { return new Prediction( predictionId, @@ -206,9 +173,6 @@ public Prediction toPrediction() { createdAt, startedAt, finishedAt, - error, - total, - succeeded, - failed); + error); } } diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java index 684b96e07786..6026e3866f5f 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java @@ -144,9 +144,6 @@ private static Prediction prediction() { 10, null, null, - null, - null, - null, null); } } diff --git a/paimon-python/pypaimon/api/prediction.py b/paimon-python/pypaimon/api/prediction.py index 69263238f109..1c7dbeb85b73 100644 --- a/paimon-python/pypaimon/api/prediction.py +++ b/paimon-python/pypaimon/api/prediction.py @@ -137,9 +137,6 @@ class GetPredictionResponse(RESTResponse): started_at: Optional[int] = optional_json_field("startedAt", "non_null") finished_at: Optional[int] = optional_json_field("finishedAt", "non_null") error: Optional[str] = optional_json_field("error", "non_null") - total: Optional[int] = optional_json_field("total", "non_null") - succeeded: Optional[int] = optional_json_field("succeeded", "non_null") - failed: Optional[int] = optional_json_field("failed", "non_null") @classmethod def from_dict(cls, data: Dict[str, Any]) -> "GetPredictionResponse": @@ -160,9 +157,6 @@ def from_dict(cls, data: Dict[str, Any]) -> "GetPredictionResponse": started_at=data.get("startedAt"), finished_at=data.get("finishedAt"), error=data.get("error"), - total=data.get("total"), - succeeded=data.get("succeeded"), - failed=data.get("failed"), ) def to_prediction(self) -> Prediction: @@ -179,9 +173,6 @@ def to_prediction(self) -> Prediction: started_at=self.started_at, finished_at=self.finished_at, error=self.error, - total=self.total, - succeeded=self.succeeded, - failed=self.failed, ) diff --git a/paimon-python/pypaimon/catalog/prediction.py b/paimon-python/pypaimon/catalog/prediction.py index d9131bbe8c9e..686dfff049c1 100644 --- a/paimon-python/pypaimon/catalog/prediction.py +++ b/paimon-python/pypaimon/catalog/prediction.py @@ -96,9 +96,6 @@ class Prediction: started_at: Optional[int] = None finished_at: Optional[int] = None error: Optional[str] = None - total: Optional[int] = None - succeeded: Optional[int] = None - failed: Optional[int] = None def __post_init__(self): _nonblank(self.prediction_id, "prediction_id") @@ -115,8 +112,7 @@ def __post_init__(self): _nonblank(self.output_table, "output_table") if type(self.created_at) is not int or self.created_at < 0: raise ValueError("created_at must be a non-negative integer") - for name in ( - "started_at", "finished_at", "total", "succeeded", "failed"): + for name in ("started_at", "finished_at"): value = getattr(self, name) if value is not None and (type(value) is not int or value < 0): raise ValueError( diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index 3d907c28bfc8..01b23340e9ea 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -160,8 +160,7 @@ def test_wait_is_client_side_polling(self): catalog = Mock() catalog.get_prediction.side_effect = [ _prediction(PredictionStatus.RUNNING), - _prediction( - PredictionStatus.SUCCEEDED, total=2, succeeded=2, failed=0), + _prediction(PredictionStatus.SUCCEEDED), ] job = PredictionJob(catalog, _prediction()) result = job.wait(timeout=timedelta(seconds=1), poll_interval=0) From 74534e1fc25f9c0b6aa3365855550857f97206e6 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Mon, 16 Nov 2026 20:37:05 +0800 Subject: [PATCH 06/41] [core][python] Model prediction lifecycle after Databricks --- docs/static/rest-catalog-open-api.yaml | 31 +++++- .../org/apache/paimon/catalog/Prediction.java | 10 +- .../paimon/catalog/PredictionStatus.java | 96 +++++++++++++++++-- .../java/org/apache/paimon/rest/RESTApi.java | 6 +- .../rest/responses/GetPredictionResponse.java | 18 +--- .../paimon/rest/RESTApiPredictionTest.java | 34 +++++-- .../org/apache/paimon/catalog/Catalog.java | 2 +- .../paimon/catalog/DelegateCatalog.java | 4 +- .../org/apache/paimon/rest/RESTCatalog.java | 4 +- .../rest/RESTCatalogPredictionTest.java | 26 +++-- paimon-python/pypaimon/api/prediction.py | 9 +- paimon-python/pypaimon/api/rest_api.py | 6 +- paimon-python/pypaimon/catalog/catalog.py | 2 +- paimon-python/pypaimon/catalog/prediction.py | 85 +++++++++++++--- .../pypaimon/catalog/rest/rest_catalog.py | 6 +- paimon-python/pypaimon/multimodal/table.py | 10 +- .../pypaimon/tests/rest/prediction_test.py | 66 +++++++++++-- 17 files changed, 314 insertions(+), 101 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index dcae938d2b44..110acd3830ec 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -174,11 +174,11 @@ paths: - {name: database, in: query, required: false, schema: {type: string}} - {name: table, in: query, required: false, schema: {type: string}} - {name: name, in: query, required: false, schema: {type: string}} - - name: status + - name: state in: query required: false schema: - $ref: '#/components/schemas/PredictionStatus' + $ref: '#/components/schemas/PredictionState' - {name: maxResults, in: query, required: false, schema: {type: integer, format: int32, minimum: 1}} - {name: pageToken, in: query, required: false, schema: {type: string}} responses: @@ -4669,9 +4669,31 @@ components: $ref: '#/components/schemas/GetModelResponse' nextPageToken: type: [ string, "null" ] - PredictionStatus: + PredictionState: + type: string + enum: [PENDING, RUNNING, TERMINATING, TERMINATED] + PredictionTerminationCode: type: string - enum: [PENDING, RUNNING, SUCCEEDED, PARTIAL_SUCCESS, FAILED, CANCELLED] + enum: [SUCCESS, SUCCESS_WITH_FAILURES, FAILED, CANCELED] + PredictionTerminationDetails: + type: object + additionalProperties: false + required: [code] + properties: + code: + $ref: '#/components/schemas/PredictionTerminationCode' + message: + type: string + PredictionStatus: + type: object + additionalProperties: false + required: [state] + properties: + state: + $ref: '#/components/schemas/PredictionState' + terminationDetails: + $ref: '#/components/schemas/PredictionTerminationDetails' + description: terminationDetails is present when state is TERMINATING or TERMINATED. PredictionSource: type: object additionalProperties: false @@ -4751,7 +4773,6 @@ components: createdAt: {type: integer, format: int64, minimum: 0} startedAt: {type: integer, format: int64, minimum: 0} finishedAt: {type: integer, format: int64, minimum: 0} - error: {type: string} ListPredictionsResponse: type: object required: [predictions] diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java index a0b4c41bd8d7..7fb8dd2fdfb7 100644 --- a/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java @@ -39,7 +39,6 @@ public class Prediction { private final long createdAt; private final @Nullable Long startedAt; private final @Nullable Long finishedAt; - private final @Nullable String error; public Prediction( String predictionId, @@ -52,8 +51,7 @@ public Prediction( String outputTable, long createdAt, @Nullable Long startedAt, - @Nullable Long finishedAt, - @Nullable String error) { + @Nullable Long finishedAt) { checkNonBlank(predictionId, "predictionId"); checkArgument(source != null, "source must not be null"); checkArgument(status != null, "status must not be null"); @@ -75,7 +73,6 @@ public Prediction( this.createdAt = createdAt; this.startedAt = startedAt; this.finishedAt = finishedAt; - this.error = error; } public String predictionId() { @@ -125,11 +122,6 @@ public Long finishedAt() { return finishedAt; } - @Nullable - public String error() { - return error; - } - private static void checkNonBlank(String value, String name) { checkArgument(value != null && !value.trim().isEmpty(), "%s must not be blank", name); } diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionStatus.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionStatus.java index b6704008f9fd..bf90ad9a2f81 100644 --- a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionStatus.java +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionStatus.java @@ -20,17 +20,95 @@ import org.apache.paimon.annotation.Experimental; -/** Lifecycle state of an asynchronous prediction job. */ +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Lifecycle and optional termination details of an asynchronous prediction. */ @Experimental -public enum PredictionStatus { - PENDING, - RUNNING, - SUCCEEDED, - PARTIAL_SUCCESS, - FAILED, - CANCELLED; +@JsonIgnoreProperties(ignoreUnknown = true) +public class PredictionStatus { + + /** Current lifecycle state. */ + public enum State { + PENDING, + RUNNING, + TERMINATING, + TERMINATED + } + + /** Result of a terminated prediction. */ + public enum TerminationCode { + SUCCESS, + SUCCESS_WITH_FAILURES, + FAILED, + CANCELED + } + + /** Result and diagnostic message available after termination. */ + @JsonIgnoreProperties(ignoreUnknown = true) + public static class TerminationDetails { + + private final TerminationCode code; + private final @Nullable String message; + + @JsonCreator + public TerminationDetails( + @JsonProperty("code") TerminationCode code, + @Nullable @JsonProperty("message") String message) { + checkArgument(code != null, "code must not be null"); + this.code = code; + this.message = message; + } + + @JsonGetter("code") + public TerminationCode code() { + return code; + } + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter("message") + public String message() { + return message; + } + } + + private final State state; + private final @Nullable TerminationDetails terminationDetails; + + @JsonCreator + public PredictionStatus( + @JsonProperty("state") State state, + @Nullable @JsonProperty("terminationDetails") TerminationDetails terminationDetails) { + checkArgument(state != null, "state must not be null"); + boolean terminating = state == State.TERMINATING || state == State.TERMINATED; + checkArgument( + terminating == (terminationDetails != null), + "terminationDetails must be present only for terminating states"); + this.state = state; + this.terminationDetails = terminationDetails; + } + + @JsonGetter("state") + public State state() { + return state; + } + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter("terminationDetails") + public TerminationDetails terminationDetails() { + return terminationDetails; + } public boolean isTerminal() { - return this == SUCCEEDED || this == PARTIAL_SUCCESS || this == FAILED || this == CANCELLED; + return state == State.TERMINATED; } } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index faf0258e22e8..7f1ebfbd7a5a 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -209,7 +209,7 @@ public class RESTApi { public static final String ENTITY_TYPE = "entityType"; public static final String ENTITY_NAME = "entityName"; public static final String PREDICTION_NAME = "name"; - public static final String PREDICTION_STATUS = "status"; + public static final String PREDICTION_STATE = "state"; public static final long TOKEN_EXPIRATION_SAFE_TIME_MILLIS = 3_600_000L; @@ -1839,7 +1839,7 @@ public GetPredictionResponse getPrediction(String predictionId) { public PagedList listPredictions( @Nullable Identifier source, @Nullable String name, - @Nullable PredictionStatus status, + @Nullable PredictionStatus.State state, @Nullable Integer maxResults, @Nullable String pageToken) { Map queryParams = @@ -1849,7 +1849,7 @@ public PagedList listPredictions( Pair.of("database", source == null ? null : source.getDatabaseName()), Pair.of("table", source == null ? null : source.getObjectName()), Pair.of(PREDICTION_NAME, name), - Pair.of(PREDICTION_STATUS, status == null ? null : status.name())); + Pair.of(PREDICTION_STATE, state == null ? null : state.name())); ListPredictionsResponse response = client.get( resourcePaths.predictions(), diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java index 642110b087c1..c48cb444f2a1 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java @@ -46,7 +46,6 @@ public class GetPredictionResponse implements RESTResponse { private final long createdAt; private final @Nullable Long startedAt; private final @Nullable Long finishedAt; - private final @Nullable String error; @JsonCreator public GetPredictionResponse( @@ -60,8 +59,7 @@ public GetPredictionResponse( @JsonProperty("outputTable") String outputTable, @JsonProperty("createdAt") long createdAt, @Nullable @JsonProperty("startedAt") Long startedAt, - @Nullable @JsonProperty("finishedAt") Long finishedAt, - @Nullable @JsonProperty("error") String error) { + @Nullable @JsonProperty("finishedAt") Long finishedAt) { this.predictionId = predictionId; this.source = source; this.name = name; @@ -73,7 +71,6 @@ public GetPredictionResponse( this.createdAt = createdAt; this.startedAt = startedAt; this.finishedAt = finishedAt; - this.error = error; } public GetPredictionResponse(Prediction prediction) { @@ -88,8 +85,7 @@ public GetPredictionResponse(Prediction prediction) { prediction.outputTable(), prediction.createdAt(), prediction.startedAt(), - prediction.finishedAt(), - prediction.error()); + prediction.finishedAt()); } @JsonGetter("predictionId") @@ -153,13 +149,6 @@ public Long getFinishedAt() { return finishedAt; } - @Nullable - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonGetter("error") - public String getError() { - return error; - } - public Prediction toPrediction() { return new Prediction( predictionId, @@ -172,7 +161,6 @@ public Prediction toPrediction() { outputTable, createdAt, startedAt, - finishedAt, - error); + finishedAt); } } diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java index 6026e3866f5f..c3e14dfa8f1d 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java @@ -60,7 +60,9 @@ void testRequestAndResponseWireFormat() throws Exception { GetPredictionResponse response = new GetPredictionResponse(prediction()); GetPredictionResponse decoded = RESTApi.fromJson(RESTApi.toJson(response), GetPredictionResponse.class); - assertThat(decoded.toPrediction().status()).isEqualTo(PredictionStatus.PENDING); + assertThat(decoded.toPrediction().status().state()) + .isEqualTo(PredictionStatus.State.PENDING); + assertThat(decoded.toPrediction().status().terminationDetails()).isNull(); assertThat(decoded.getName()).isNull(); ListPredictionsResponse empty = RESTApi.fromJson("{}", ListPredictionsResponse.class); @@ -107,12 +109,27 @@ void testSpecValidation() { @Test void testTerminalStates() { - assertThat(PredictionStatus.PENDING.isTerminal()).isFalse(); - assertThat(PredictionStatus.RUNNING.isTerminal()).isFalse(); - assertThat(PredictionStatus.SUCCEEDED.isTerminal()).isTrue(); - assertThat(PredictionStatus.PARTIAL_SUCCESS.isTerminal()).isTrue(); - assertThat(PredictionStatus.FAILED.isTerminal()).isTrue(); - assertThat(PredictionStatus.CANCELLED.isTerminal()).isTrue(); + assertThat(new PredictionStatus(PredictionStatus.State.PENDING, null).isTerminal()) + .isFalse(); + assertThat(new PredictionStatus(PredictionStatus.State.RUNNING, null).isTerminal()) + .isFalse(); + assertThat( + new PredictionStatus( + PredictionStatus.State.TERMINATING, + new PredictionStatus.TerminationDetails( + PredictionStatus.TerminationCode.CANCELED, null)) + .isTerminal()) + .isFalse(); + PredictionStatus terminated = + new PredictionStatus( + PredictionStatus.State.TERMINATED, + new PredictionStatus.TerminationDetails( + PredictionStatus.TerminationCode.SUCCESS, null)); + assertThat(terminated.isTerminal()).isTrue(); + assertThat(terminated.terminationDetails().code()) + .isEqualTo(PredictionStatus.TerminationCode.SUCCESS); + assertThatThrownBy(() -> new PredictionStatus(PredictionStatus.State.TERMINATED, null)) + .isInstanceOf(IllegalArgumentException.class); } private static PredictionSpec spec() { @@ -136,14 +153,13 @@ private static Prediction prediction() { "prediction-1", new PredictionSource("quality", "clips"), null, - PredictionStatus.PENDING, + new PredictionStatus(PredictionStatus.State.PENDING, null), 7, "quality.qwen_vl", "1", "system.prediction_results", 10, null, - null, null); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java index b3e7f836dd7e..d3bac00bfb01 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java @@ -1214,7 +1214,7 @@ default Prediction getPrediction(String predictionId) throws PredictionNotExistE default PagedList listPredictions( @Nullable Identifier source, @Nullable String name, - @Nullable PredictionStatus status, + @Nullable PredictionStatus.State state, @Nullable Integer maxResults, @Nullable String pageToken) { throw new UnsupportedOperationException("Current Catalog does not support predictions"); diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java index b4183cc30552..60df74b60b9d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java @@ -422,10 +422,10 @@ public Prediction getPrediction(String predictionId) throws PredictionNotExistEx public PagedList listPredictions( @Nullable Identifier source, @Nullable String name, - @Nullable PredictionStatus status, + @Nullable PredictionStatus.State state, @Nullable Integer maxResults, @Nullable String pageToken) { - return wrapped.listPredictions(source, name, status, maxResults, pageToken); + return wrapped.listPredictions(source, name, state, maxResults, pageToken); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 0bec83191dac..0cdad8ed297f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -1245,12 +1245,12 @@ public Prediction getPrediction(String predictionId) throws PredictionNotExistEx public PagedList listPredictions( @Nullable Identifier source, @Nullable String name, - @Nullable PredictionStatus status, + @Nullable PredictionStatus.State state, @Nullable Integer maxResults, @Nullable String pageToken) { try { PagedList response = - api.listPredictions(source, name, status, maxResults, pageToken); + api.listPredictions(source, name, state, maxResults, pageToken); return new PagedList<>( response.getElements().stream() .map( diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java index efab562c56d1..78137be684c7 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java @@ -69,7 +69,7 @@ void testPredictionLifecycleUsesSharedRestContract() throws Exception { 200)); PagedList page = catalog.listPredictions( - table, "daily-quality", PredictionStatus.RUNNING, 10, "page-1"); + table, "daily-quality", PredictionStatus.State.RUNNING, 10, "page-1"); assertThat(page.getElements()) .extracting(Prediction::predictionId) .containsExactly("prediction-1"); @@ -79,13 +79,15 @@ void testPredictionLifecycleUsesSharedRestContract() throws Exception { .contains("database=quality") .contains("table=clips") .contains("name=daily-quality") - .contains("status=RUNNING") + .contains("state=RUNNING") .contains("maxResults=10") .contains("pageToken=page-1"); - server.enqueue(json(predictionJson("CANCELLED"), 200)); - assertThat(catalog.cancelPrediction("prediction/1").status()) - .isEqualTo(PredictionStatus.CANCELLED); + server.enqueue(json(terminatedPredictionJson("CANCELED"), 200)); + PredictionStatus cancelled = catalog.cancelPrediction("prediction/1").status(); + assertThat(cancelled.state()).isEqualTo(PredictionStatus.State.TERMINATED); + assertThat(cancelled.terminationDetails().code()) + .isEqualTo(PredictionStatus.TerminationCode.CANCELED); assertThat(server.takeRequest(10, TimeUnit.SECONDS).getPath()) .isEqualTo("/v1/catalog/predictions/prediction%2F1/cancel"); } @@ -133,9 +135,19 @@ private static Catalog catalog(MockWebServer server) { private static String predictionJson(String status) { return "{\"predictionId\":\"prediction-1\",\"source\":{\"database\":\"quality\"," - + "\"table\":\"clips\"},\"name\":\"daily-quality\",\"status\":\"" + + "\"table\":\"clips\"},\"name\":\"daily-quality\",\"status\":{\"state\":\"" + status - + "\",\"snapshotId\":7,\"model\":\"quality.qwen_vl\"," + + "\"},\"snapshotId\":7,\"model\":\"quality.qwen_vl\"," + + "\"modelVersion\":\"1\",\"outputTable\":\"system.results\"," + + "\"createdAt\":10}"; + } + + private static String terminatedPredictionJson(String code) { + return "{\"predictionId\":\"prediction-1\",\"source\":{\"database\":\"quality\"," + + "\"table\":\"clips\"},\"name\":\"daily-quality\"," + + "\"status\":{\"state\":\"TERMINATED\",\"terminationDetails\":{\"code\":\"" + + code + + "\"}},\"snapshotId\":7,\"model\":\"quality.qwen_vl\"," + "\"modelVersion\":\"1\",\"outputTable\":\"system.results\"," + "\"createdAt\":10}"; } diff --git a/paimon-python/pypaimon/api/prediction.py b/paimon-python/pypaimon/api/prediction.py index 1c7dbeb85b73..ce2e4334b59b 100644 --- a/paimon-python/pypaimon/api/prediction.py +++ b/paimon-python/pypaimon/api/prediction.py @@ -127,7 +127,7 @@ class CancelPredictionRequest(RESTRequest): class GetPredictionResponse(RESTResponse): prediction_id: str = json_field("predictionId") source: PredictionSource = None - status: str = "" + status: PredictionStatus = None snapshot_id: int = json_field("snapshotId", default=0) model: str = "" model_version: str = json_field("modelVersion", default="") @@ -136,7 +136,6 @@ class GetPredictionResponse(RESTResponse): name: Optional[str] = optional_json_field("name", "non_null") started_at: Optional[int] = optional_json_field("startedAt", "non_null") finished_at: Optional[int] = optional_json_field("finishedAt", "non_null") - error: Optional[str] = optional_json_field("error", "non_null") @classmethod def from_dict(cls, data: Dict[str, Any]) -> "GetPredictionResponse": @@ -148,7 +147,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "GetPredictionResponse": else data.get("source") ), name=data.get("name"), - status=data.get("status"), + status=PredictionStatus.from_dict(data.get("status")), snapshot_id=data.get("snapshotId"), model=data.get("model"), model_version=data.get("modelVersion"), @@ -156,7 +155,6 @@ def from_dict(cls, data: Dict[str, Any]) -> "GetPredictionResponse": created_at=data.get("createdAt"), started_at=data.get("startedAt"), finished_at=data.get("finishedAt"), - error=data.get("error"), ) def to_prediction(self) -> Prediction: @@ -164,7 +162,7 @@ def to_prediction(self) -> Prediction: prediction_id=self.prediction_id, source=self.source.to_identifier(), name=self.name, - status=PredictionStatus(self.status), + status=self.status, snapshot_id=self.snapshot_id, model=self.model, model_version=self.model_version, @@ -172,7 +170,6 @@ def to_prediction(self) -> Prediction: created_at=self.created_at, started_at=self.started_at, finished_at=self.finished_at, - error=self.error, ) diff --git a/paimon-python/pypaimon/api/rest_api.py b/paimon-python/pypaimon/api/rest_api.py index f68c1856575e..84d4910aeb23 100755 --- a/paimon-python/pypaimon/api/rest_api.py +++ b/paimon-python/pypaimon/api/rest_api.py @@ -65,7 +65,7 @@ from pypaimon.common.options import Options from pypaimon.common.options.config import CatalogOptions from pypaimon.common.identifier import Identifier -from pypaimon.catalog.prediction import PredictionStatus +from pypaimon.catalog.prediction import PredictionState from pypaimon.management.data_policy import DataPolicy from pypaimon.management.list_permissions_request import \ ListPermissionsRequest @@ -812,7 +812,7 @@ def list_predictions( self, source: Optional[Identifier] = None, name: Optional[str] = None, - status: Optional[PredictionStatus] = None, + state: Optional[PredictionState] = None, max_results: Optional[int] = None, page_token: Optional[str] = None, ) -> PagedList[GetPredictionResponse]: @@ -823,7 +823,7 @@ def list_predictions( self._put_query_parameter(query_params, "table", table_name) self._put_query_parameter(query_params, "name", name) self._put_query_parameter( - query_params, "status", status.value if status is not None else None) + query_params, "state", state.value if state is not None else None) if max_results is not None and max_results > 0: query_params[self.MAX_RESULTS] = str(max_results) self._put_query_parameter(query_params, self.PAGE_TOKEN, page_token) diff --git a/paimon-python/pypaimon/catalog/catalog.py b/paimon-python/pypaimon/catalog/catalog.py index dbe670113e67..0fab39ac6c4c 100644 --- a/paimon-python/pypaimon/catalog/catalog.py +++ b/paimon-python/pypaimon/catalog/catalog.py @@ -132,7 +132,7 @@ def list_predictions( self, source: Optional[Union[str, Identifier]] = None, name: Optional[str] = None, - status: Optional['PredictionStatus'] = None, + state: Optional['PredictionState'] = None, max_results: Optional[int] = None, page_token: Optional[str] = None, ) -> PagedList['Prediction']: diff --git a/paimon-python/pypaimon/catalog/prediction.py b/paimon-python/pypaimon/catalog/prediction.py index 686dfff049c1..256098cc4ff2 100644 --- a/paimon-python/pypaimon/catalog/prediction.py +++ b/paimon-python/pypaimon/catalog/prediction.py @@ -20,24 +20,86 @@ from typing import Any, Dict, List, Optional from pypaimon.common.identifier import Identifier +from pypaimon.common.json_util import optional_json_field -class PredictionStatus(str, Enum): +class PredictionState(str, Enum): PENDING = "PENDING" RUNNING = "RUNNING" - SUCCEEDED = "SUCCEEDED" - PARTIAL_SUCCESS = "PARTIAL_SUCCESS" + TERMINATING = "TERMINATING" + TERMINATED = "TERMINATED" + + +class PredictionTerminationCode(str, Enum): + SUCCESS = "SUCCESS" + SUCCESS_WITH_FAILURES = "SUCCESS_WITH_FAILURES" FAILED = "FAILED" - CANCELLED = "CANCELLED" + CANCELED = "CANCELED" + + +@dataclass +class PredictionTerminationDetails: + """Result and diagnostic message available after termination.""" + + code: PredictionTerminationCode + message: Optional[str] = None + + def __post_init__(self): + if isinstance(self.code, str): + self.code = PredictionTerminationCode(self.code) + if not isinstance(self.code, PredictionTerminationCode): + raise ValueError("code must be a PredictionTerminationCode") + + +@dataclass +class PredictionStatus: + """Lifecycle and optional termination details of a prediction.""" + + state: PredictionState + termination_details: Optional[PredictionTerminationDetails] = \ + optional_json_field("terminationDetails", "non_null") + + def __post_init__(self): + if isinstance(self.state, str): + self.state = PredictionState(self.state) + if not isinstance(self.state, PredictionState): + raise ValueError("state must be a PredictionState") + terminating = self.state in { + PredictionState.TERMINATING, + PredictionState.TERMINATED, + } + if terminating != (self.termination_details is not None): + raise ValueError( + "termination_details must be present only for terminating states") + if self.termination_details is not None and not isinstance( + self.termination_details, PredictionTerminationDetails): + raise ValueError( + "termination_details must be PredictionTerminationDetails") + + @classmethod + def from_dict(cls, value): + details = value.get("terminationDetails") + return cls( + state=value.get("state"), + termination_details=( + PredictionTerminationDetails(**details) + if details is not None else None), + ) + + def to_dict(self): + result = {"state": self.state.value} + if self.termination_details is not None: + result["terminationDetails"] = { + "code": self.termination_details.code.value, + } + if self.termination_details.message is not None: + result["terminationDetails"]["message"] = \ + self.termination_details.message + return result @property def terminal(self) -> bool: - return self in { - PredictionStatus.SUCCEEDED, - PredictionStatus.PARTIAL_SUCCESS, - PredictionStatus.FAILED, - PredictionStatus.CANCELLED, - } + return self.state is PredictionState.TERMINATED @dataclass @@ -95,14 +157,11 @@ class Prediction: name: Optional[str] = None started_at: Optional[int] = None finished_at: Optional[int] = None - error: Optional[str] = None def __post_init__(self): _nonblank(self.prediction_id, "prediction_id") if not isinstance(self.source, Identifier): raise ValueError("source must be an Identifier") - if isinstance(self.status, str): - self.status = PredictionStatus(self.status) if not isinstance(self.status, PredictionStatus): raise ValueError("status must be a PredictionStatus") if type(self.snapshot_id) is not int or self.snapshot_id <= 0: diff --git a/paimon-python/pypaimon/catalog/rest/rest_catalog.py b/paimon-python/pypaimon/catalog/rest/rest_catalog.py index a30bbdfd112f..afdbaaffaa3b 100644 --- a/paimon-python/pypaimon/catalog/rest/rest_catalog.py +++ b/paimon-python/pypaimon/catalog/rest/rest_catalog.py @@ -46,7 +46,7 @@ ) from pypaimon.catalog.database import Database from pypaimon.catalog.model import Model, ModelDefinition -from pypaimon.catalog.prediction import Prediction, PredictionSpec, PredictionStatus +from pypaimon.catalog.prediction import Prediction, PredictionSpec, PredictionState from pypaimon.catalog.rest.property_change import PropertyChange from pypaimon.catalog.rest.rest_token_file_io import RESTTokenFileIO from pypaimon.catalog.rest.table_metadata import TableMetadata @@ -717,7 +717,7 @@ def list_predictions( self, source: Optional[Union[str, Identifier]] = None, name: Optional[str] = None, - status: Optional[PredictionStatus] = None, + state: Optional[PredictionState] = None, max_results: Optional[int] = None, page_token: Optional[str] = None, ) -> PagedList[Prediction]: @@ -725,7 +725,7 @@ def list_predictions( source = Identifier.from_string(source) try: page = self.rest_api.list_predictions( - source, name, status, max_results, page_token) + source, name, state, max_results, page_token) return PagedList( [value.to_prediction() for value in page.elements], page.next_page_token, diff --git a/paimon-python/pypaimon/multimodal/table.py b/paimon-python/pypaimon/multimodal/table.py index d429630f5b71..c872a374150e 100644 --- a/paimon-python/pypaimon/multimodal/table.py +++ b/paimon-python/pypaimon/multimodal/table.py @@ -167,20 +167,20 @@ def list_predictions( self, *, name=None, - status=None, + state=None, max_results=None, page_token=None): """List prediction handles for this table.""" from pypaimon.api.api_response import PagedList - from pypaimon.catalog.prediction import PredictionStatus + from pypaimon.catalog.prediction import PredictionState from pypaimon.multimodal.prediction import PredictionJob - if isinstance(status, str): - status = PredictionStatus(status) + if isinstance(state, str): + state = PredictionState(state) page = self.catalog.list_predictions( source=self.raw_table.identifier, name=name, - status=status, + state=state, max_results=max_results, page_token=page_token, ) diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index 01b23340e9ea..64332afb6f3c 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -31,8 +31,11 @@ from pypaimon.api.rest_api import RESTApi from pypaimon.catalog.prediction import ( Prediction, + PredictionState, PredictionSpec, PredictionStatus, + PredictionTerminationCode, + PredictionTerminationDetails, ) from pypaimon.catalog.rest.rest_catalog import RESTCatalog from pypaimon.common.identifier import Identifier @@ -56,7 +59,9 @@ def _spec(**changes): return PredictionSpec(**values) -def _prediction(status=PredictionStatus.PENDING, **changes): +def _prediction(status=None, **changes): + if status is None: + status = PredictionStatus(PredictionState.PENDING) values = dict( prediction_id="prediction-1", source=Identifier.create("quality", "clips"), @@ -72,7 +77,9 @@ def _prediction(status=PredictionStatus.PENDING, **changes): return Prediction(**values) -def _response(status="PENDING", **changes): +def _response(status=None, **changes): + if status is None: + status = PredictionStatus(PredictionState.PENDING) values = dict( prediction_id="prediction-1", source=PredictionSource("quality", "clips"), @@ -108,7 +115,34 @@ def test_request_and_response_wire_format(self): response = JSON.from_json( JSON.to_json(_response()), GetPredictionResponse) self.assertEqual( - PredictionStatus.PENDING, response.to_prediction().status) + PredictionState.PENDING, response.to_prediction().status.state) + self.assertIsNone( + response.to_prediction().status.termination_details) + terminated = PredictionStatus( + PredictionState.TERMINATED, + PredictionTerminationDetails( + PredictionTerminationCode.FAILED, + "Model request failed", + ), + ) + terminated_body = json.loads(JSON.to_json(_response(terminated))) + self.assertEqual( + { + "state": "TERMINATED", + "terminationDetails": { + "code": "FAILED", + "message": "Model request failed", + }, + }, + terminated_body["status"], + ) + self.assertEqual( + terminated, + JSON.from_json( + JSON.to_json(_response(terminated)), + GetPredictionResponse, + ).status, + ) page = JSON.from_json( JSON.to_json(ListPredictionsResponse([_response()], "next")), ListPredictionsResponse, @@ -131,6 +165,14 @@ def test_validation(self): for changes in invalid: with self.subTest(changes=changes), self.assertRaises(ValueError): _spec(**changes) + with self.assertRaises(ValueError): + PredictionStatus(PredictionState.TERMINATED) + with self.assertRaises(ValueError): + PredictionStatus( + PredictionState.RUNNING, + PredictionTerminationDetails( + PredictionTerminationCode.FAILED), + ) def test_table_submit_fixes_view_and_returns_job(self): raw = Mock(spec=FileStoreTable) @@ -159,13 +201,21 @@ def test_table_submit_fixes_view_and_returns_job(self): def test_wait_is_client_side_polling(self): catalog = Mock() catalog.get_prediction.side_effect = [ - _prediction(PredictionStatus.RUNNING), - _prediction(PredictionStatus.SUCCEEDED), + _prediction(PredictionStatus(PredictionState.RUNNING)), + _prediction(PredictionStatus( + PredictionState.TERMINATED, + PredictionTerminationDetails( + PredictionTerminationCode.SUCCESS), + )), ] job = PredictionJob(catalog, _prediction()) result = job.wait(timeout=timedelta(seconds=1), poll_interval=0) self.assertIs(job, result) - self.assertEqual(PredictionStatus.SUCCEEDED, job.status) + self.assertEqual(PredictionState.TERMINATED, job.status.state) + self.assertEqual( + PredictionTerminationCode.SUCCESS, + job.status.termination_details.code, + ) self.assertEqual(2, catalog.get_prediction.call_count) def test_rest_round_trip_paths_and_filters(self): @@ -229,7 +279,7 @@ def do_GET(self): page = catalog.list_predictions( identifier, name="daily-quality", - status=PredictionStatus.RUNNING, + state=PredictionState.RUNNING, max_results=10, page_token="p1", ) @@ -237,7 +287,7 @@ def do_GET(self): self.assertIn("name=daily-quality", calls[-1][1]) self.assertIn("database=a%2Fb", calls[-1][1]) self.assertIn("table=clip+table", calls[-1][1]) - self.assertIn("status=RUNNING", calls[-1][1]) + self.assertIn("state=RUNNING", calls[-1][1]) self.assertIn("maxResults=10", calls[-1][1]) self.assertIn("pageToken=p1", calls[-1][1]) From aa129160d26dc731a51ece833259d0ddc7989e35 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Mon, 16 Nov 2026 20:51:20 +0800 Subject: [PATCH 07/41] [core][python] Align prediction times with Databricks --- docs/static/rest-catalog-open-api.yaml | 7 ++-- .../org/apache/paimon/catalog/Prediction.java | 33 ++++++--------- .../rest/responses/GetPredictionResponse.java | 42 +++++++------------ .../paimon/rest/RESTApiPredictionTest.java | 24 ++++++++++- .../rest/RESTCatalogPredictionTest.java | 5 +-- paimon-python/pypaimon/api/prediction.py | 18 ++++---- paimon-python/pypaimon/catalog/prediction.py | 12 +++--- .../pypaimon/tests/rest/prediction_test.py | 14 +++++-- 8 files changed, 80 insertions(+), 75 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 110acd3830ec..3d8fe272521f 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -4758,7 +4758,7 @@ components: GetPredictionResponse: type: object additionalProperties: false - required: [predictionId, source, status, snapshotId, model, modelVersion, outputTable, createdAt] + required: [predictionId, source, status, snapshotId, model, modelVersion, outputTable] properties: predictionId: {type: string, minLength: 1} source: @@ -4770,9 +4770,8 @@ components: model: {type: string, minLength: 1} modelVersion: {type: string, minLength: 1} outputTable: {type: string, minLength: 1} - createdAt: {type: integer, format: int64, minimum: 0} - startedAt: {type: integer, format: int64, minimum: 0} - finishedAt: {type: integer, format: int64, minimum: 0} + startTime: {type: integer, format: int64, minimum: 0} + endTime: {type: integer, format: int64, minimum: 0} ListPredictionsResponse: type: object required: [predictions] diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java index 7fb8dd2fdfb7..d962761c72ea 100644 --- a/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java @@ -36,9 +36,8 @@ public class Prediction { private final String model; private final String modelVersion; private final String outputTable; - private final long createdAt; - private final @Nullable Long startedAt; - private final @Nullable Long finishedAt; + private final @Nullable Long startTime; + private final @Nullable Long endTime; public Prediction( String predictionId, @@ -49,9 +48,8 @@ public Prediction( String model, String modelVersion, String outputTable, - long createdAt, - @Nullable Long startedAt, - @Nullable Long finishedAt) { + @Nullable Long startTime, + @Nullable Long endTime) { checkNonBlank(predictionId, "predictionId"); checkArgument(source != null, "source must not be null"); checkArgument(status != null, "status must not be null"); @@ -59,9 +57,8 @@ public Prediction( checkNonBlank(model, "model"); checkNonBlank(modelVersion, "modelVersion"); checkNonBlank(outputTable, "outputTable"); - checkArgument(createdAt >= 0, "createdAt must not be negative"); - checkOptionalNonNegative(startedAt, "startedAt"); - checkOptionalNonNegative(finishedAt, "finishedAt"); + checkOptionalNonNegative(startTime, "startTime"); + checkOptionalNonNegative(endTime, "endTime"); this.predictionId = predictionId; this.source = source; this.name = name; @@ -70,9 +67,8 @@ public Prediction( this.model = model; this.modelVersion = modelVersion; this.outputTable = outputTable; - this.createdAt = createdAt; - this.startedAt = startedAt; - this.finishedAt = finishedAt; + this.startTime = startTime; + this.endTime = endTime; } public String predictionId() { @@ -108,18 +104,13 @@ public String outputTable() { return outputTable; } - public long createdAt() { - return createdAt; + public Long startTime() { + return startTime; } @Nullable - public Long startedAt() { - return startedAt; - } - - @Nullable - public Long finishedAt() { - return finishedAt; + public Long endTime() { + return endTime; } private static void checkNonBlank(String value, String name) { diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java index c48cb444f2a1..e8bd656bded0 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java @@ -43,9 +43,8 @@ public class GetPredictionResponse implements RESTResponse { private final String model; private final String modelVersion; private final String outputTable; - private final long createdAt; - private final @Nullable Long startedAt; - private final @Nullable Long finishedAt; + private final @Nullable Long startTime; + private final @Nullable Long endTime; @JsonCreator public GetPredictionResponse( @@ -57,9 +56,8 @@ public GetPredictionResponse( @JsonProperty("model") String model, @JsonProperty("modelVersion") String modelVersion, @JsonProperty("outputTable") String outputTable, - @JsonProperty("createdAt") long createdAt, - @Nullable @JsonProperty("startedAt") Long startedAt, - @Nullable @JsonProperty("finishedAt") Long finishedAt) { + @Nullable @JsonProperty("startTime") Long startTime, + @Nullable @JsonProperty("endTime") Long endTime) { this.predictionId = predictionId; this.source = source; this.name = name; @@ -68,9 +66,8 @@ public GetPredictionResponse( this.model = model; this.modelVersion = modelVersion; this.outputTable = outputTable; - this.createdAt = createdAt; - this.startedAt = startedAt; - this.finishedAt = finishedAt; + this.startTime = startTime; + this.endTime = endTime; } public GetPredictionResponse(Prediction prediction) { @@ -83,9 +80,8 @@ public GetPredictionResponse(Prediction prediction) { prediction.model(), prediction.modelVersion(), prediction.outputTable(), - prediction.createdAt(), - prediction.startedAt(), - prediction.finishedAt()); + prediction.startTime(), + prediction.endTime()); } @JsonGetter("predictionId") @@ -130,23 +126,18 @@ public String getOutputTable() { return outputTable; } - @JsonGetter("createdAt") - public long getCreatedAt() { - return createdAt; - } - @Nullable @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonGetter("startedAt") - public Long getStartedAt() { - return startedAt; + @JsonGetter("startTime") + public Long getStartTime() { + return startTime; } @Nullable @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonGetter("finishedAt") - public Long getFinishedAt() { - return finishedAt; + @JsonGetter("endTime") + public Long getEndTime() { + return endTime; } public Prediction toPrediction() { @@ -159,8 +150,7 @@ public Prediction toPrediction() { model, modelVersion, outputTable, - createdAt, - startedAt, - finishedAt); + startTime, + endTime); } } diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java index c3e14dfa8f1d..4256b990e4a4 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java @@ -64,6 +64,29 @@ void testRequestAndResponseWireFormat() throws Exception { .isEqualTo(PredictionStatus.State.PENDING); assertThat(decoded.toPrediction().status().terminationDetails()).isNull(); assertThat(decoded.getName()).isNull(); + assertThat(RESTApi.fromJson(RESTApi.toJson(response), Map.class)) + .doesNotContainKeys("createdAt", "startedAt", "finishedAt", "startTime", "endTime"); + + GetPredictionResponse completed = + new GetPredictionResponse( + new Prediction( + "prediction-2", + new PredictionSource("quality", "clips"), + null, + new PredictionStatus( + PredictionStatus.State.TERMINATED, + new PredictionStatus.TerminationDetails( + PredictionStatus.TerminationCode.SUCCESS, null)), + 7, + "quality.qwen_vl", + "1", + "system.prediction_results", + 10L, + 20L)); + assertThat(RESTApi.fromJson(RESTApi.toJson(completed), Map.class)) + .containsEntry("startTime", 10) + .containsEntry("endTime", 20) + .doesNotContainKeys("createdAt", "startedAt", "finishedAt"); ListPredictionsResponse empty = RESTApi.fromJson("{}", ListPredictionsResponse.class); assertThat(empty.getPredictions()).isEmpty(); @@ -158,7 +181,6 @@ private static Prediction prediction() { "quality.qwen_vl", "1", "system.prediction_results", - 10, null, null); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java index 78137be684c7..d9e5e2018068 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java @@ -138,8 +138,7 @@ private static String predictionJson(String status) { + "\"table\":\"clips\"},\"name\":\"daily-quality\",\"status\":{\"state\":\"" + status + "\"},\"snapshotId\":7,\"model\":\"quality.qwen_vl\"," - + "\"modelVersion\":\"1\",\"outputTable\":\"system.results\"," - + "\"createdAt\":10}"; + + "\"modelVersion\":\"1\",\"outputTable\":\"system.results\"}"; } private static String terminatedPredictionJson(String code) { @@ -149,7 +148,7 @@ private static String terminatedPredictionJson(String code) { + code + "\"}},\"snapshotId\":7,\"model\":\"quality.qwen_vl\"," + "\"modelVersion\":\"1\",\"outputTable\":\"system.results\"," - + "\"createdAt\":10}"; + + "\"startTime\":10,\"endTime\":20}"; } private static MockResponse json(String body, int code) { diff --git a/paimon-python/pypaimon/api/prediction.py b/paimon-python/pypaimon/api/prediction.py index ce2e4334b59b..8cd545f46ba6 100644 --- a/paimon-python/pypaimon/api/prediction.py +++ b/paimon-python/pypaimon/api/prediction.py @@ -44,7 +44,8 @@ def __post_init__(self): @classmethod def from_identifier(cls, identifier: Identifier) -> "PredictionSource": - return cls(identifier.get_database_name(), identifier.get_object_name()) + return cls( + identifier.get_database_name(), identifier.get_object_name()) def to_identifier(self) -> Identifier: return Identifier.create(self.database, self.table) @@ -132,10 +133,9 @@ class GetPredictionResponse(RESTResponse): model: str = "" model_version: str = json_field("modelVersion", default="") output_table: str = json_field("outputTable", default="") - created_at: int = json_field("createdAt", default=0) name: Optional[str] = optional_json_field("name", "non_null") - started_at: Optional[int] = optional_json_field("startedAt", "non_null") - finished_at: Optional[int] = optional_json_field("finishedAt", "non_null") + start_time: Optional[int] = optional_json_field("startTime", "non_null") + end_time: Optional[int] = optional_json_field("endTime", "non_null") @classmethod def from_dict(cls, data: Dict[str, Any]) -> "GetPredictionResponse": @@ -152,9 +152,8 @@ def from_dict(cls, data: Dict[str, Any]) -> "GetPredictionResponse": model=data.get("model"), model_version=data.get("modelVersion"), output_table=data.get("outputTable"), - created_at=data.get("createdAt"), - started_at=data.get("startedAt"), - finished_at=data.get("finishedAt"), + start_time=data.get("startTime"), + end_time=data.get("endTime"), ) def to_prediction(self) -> Prediction: @@ -167,9 +166,8 @@ def to_prediction(self) -> Prediction: model=self.model, model_version=self.model_version, output_table=self.output_table, - created_at=self.created_at, - started_at=self.started_at, - finished_at=self.finished_at, + start_time=self.start_time, + end_time=self.end_time, ) diff --git a/paimon-python/pypaimon/catalog/prediction.py b/paimon-python/pypaimon/catalog/prediction.py index 256098cc4ff2..e9aae7df5acf 100644 --- a/paimon-python/pypaimon/catalog/prediction.py +++ b/paimon-python/pypaimon/catalog/prediction.py @@ -70,7 +70,8 @@ def __post_init__(self): } if terminating != (self.termination_details is not None): raise ValueError( - "termination_details must be present only for terminating states") + "termination_details must be present only for " + "terminating states") if self.termination_details is not None and not isinstance( self.termination_details, PredictionTerminationDetails): raise ValueError( @@ -153,10 +154,9 @@ class Prediction: model: str model_version: str output_table: str - created_at: int name: Optional[str] = None - started_at: Optional[int] = None - finished_at: Optional[int] = None + start_time: Optional[int] = None + end_time: Optional[int] = None def __post_init__(self): _nonblank(self.prediction_id, "prediction_id") @@ -169,9 +169,7 @@ def __post_init__(self): _nonblank(self.model, "model") _nonblank(self.model_version, "model_version") _nonblank(self.output_table, "output_table") - if type(self.created_at) is not int or self.created_at < 0: - raise ValueError("created_at must be a non-negative integer") - for name in ("started_at", "finished_at"): + for name in ("start_time", "end_time"): value = getattr(self, name) if value is not None and (type(value) is not int or value < 0): raise ValueError( diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index 64332afb6f3c..d16e2d84491b 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -71,7 +71,6 @@ def _prediction(status=None, **changes): model="quality.qwen_vl", model_version="1", output_table="system.prediction_results", - created_at=10, ) values.update(changes) return Prediction(**values) @@ -89,7 +88,6 @@ def _response(status=None, **changes): model="quality.qwen_vl", model_version="1", output_table="system.prediction_results", - created_at=10, ) values.update(changes) return GetPredictionResponse(**values) @@ -118,6 +116,11 @@ def test_request_and_response_wire_format(self): PredictionState.PENDING, response.to_prediction().status.state) self.assertIsNone( response.to_prediction().status.termination_details) + pending_body = json.loads(JSON.to_json(_response())) + for field in ( + "createdAt", "startedAt", "finishedAt", + "startTime", "endTime"): + self.assertNotIn(field, pending_body) terminated = PredictionStatus( PredictionState.TERMINATED, PredictionTerminationDetails( @@ -125,7 +128,8 @@ def test_request_and_response_wire_format(self): "Model request failed", ), ) - terminated_body = json.loads(JSON.to_json(_response(terminated))) + terminated_body = json.loads(JSON.to_json(_response( + terminated, start_time=10, end_time=20))) self.assertEqual( { "state": "TERMINATED", @@ -136,6 +140,10 @@ def test_request_and_response_wire_format(self): }, terminated_body["status"], ) + self.assertEqual(10, terminated_body["startTime"]) + self.assertEqual(20, terminated_body["endTime"]) + for field in ("createdAt", "startedAt", "finishedAt"): + self.assertNotIn(field, terminated_body) self.assertEqual( terminated, JSON.from_json( From 4dbc8eb456a1a85f37f11d09cdd73503adb44ab5 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Mon, 16 Nov 2026 21:05:56 +0800 Subject: [PATCH 08/41] [core][python] Make prediction output extensible --- docs/static/rest-catalog-open-api.yaml | 12 ++++- .../org/apache/paimon/catalog/Prediction.java | 13 ++--- .../paimon/catalog/PredictionOutput.java | 49 +++++++++++++++++++ .../rest/responses/GetPredictionResponse.java | 19 ++++--- .../paimon/rest/RESTApiPredictionTest.java | 24 +++++++-- .../rest/RESTCatalogPredictionTest.java | 5 +- paimon-python/pypaimon/api/prediction.py | 36 ++++++++++++-- paimon-python/pypaimon/catalog/prediction.py | 17 ++++++- .../pypaimon/tests/rest/prediction_test.py | 34 +++++++++---- 9 files changed, 171 insertions(+), 38 deletions(-) create mode 100644 paimon-api/src/main/java/org/apache/paimon/catalog/PredictionOutput.java diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 3d8fe272521f..22668929eeff 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -4701,6 +4701,13 @@ components: properties: database: {type: string, minLength: 1} table: {type: string, minLength: 1} + PredictionOutput: + type: object + minProperties: 1 + properties: + table: + $ref: '#/components/schemas/PredictionSource' + additionalProperties: true SubmitPredictionRequest: type: object additionalProperties: false @@ -4758,7 +4765,7 @@ components: GetPredictionResponse: type: object additionalProperties: false - required: [predictionId, source, status, snapshotId, model, modelVersion, outputTable] + required: [predictionId, source, status, snapshotId, model, modelVersion] properties: predictionId: {type: string, minLength: 1} source: @@ -4769,7 +4776,8 @@ components: snapshotId: {type: integer, format: int64, minimum: 1} model: {type: string, minLength: 1} modelVersion: {type: string, minLength: 1} - outputTable: {type: string, minLength: 1} + output: + $ref: '#/components/schemas/PredictionOutput' startTime: {type: integer, format: int64, minimum: 0} endTime: {type: integer, format: int64, minimum: 0} ListPredictionsResponse: diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java index d962761c72ea..c826f8082b45 100644 --- a/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java @@ -35,7 +35,7 @@ public class Prediction { private final long snapshotId; private final String model; private final String modelVersion; - private final String outputTable; + private final @Nullable PredictionOutput output; private final @Nullable Long startTime; private final @Nullable Long endTime; @@ -47,7 +47,7 @@ public Prediction( long snapshotId, String model, String modelVersion, - String outputTable, + @Nullable PredictionOutput output, @Nullable Long startTime, @Nullable Long endTime) { checkNonBlank(predictionId, "predictionId"); @@ -56,7 +56,6 @@ public Prediction( checkArgument(snapshotId > 0, "snapshotId must be positive"); checkNonBlank(model, "model"); checkNonBlank(modelVersion, "modelVersion"); - checkNonBlank(outputTable, "outputTable"); checkOptionalNonNegative(startTime, "startTime"); checkOptionalNonNegative(endTime, "endTime"); this.predictionId = predictionId; @@ -66,7 +65,7 @@ public Prediction( this.snapshotId = snapshotId; this.model = model; this.modelVersion = modelVersion; - this.outputTable = outputTable; + this.output = output; this.startTime = startTime; this.endTime = endTime; } @@ -100,10 +99,12 @@ public String modelVersion() { return modelVersion; } - public String outputTable() { - return outputTable; + @Nullable + public PredictionOutput output() { + return output; } + @Nullable public Long startTime() { return startTime; } diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionOutput.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionOutput.java new file mode 100644 index 000000000000..0097df3d26a4 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionOutput.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.catalog; + +import org.apache.paimon.annotation.Experimental; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +/** Output of an asynchronous prediction. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class PredictionOutput { + + private final @Nullable PredictionSource table; + + @JsonCreator + public PredictionOutput(@Nullable @JsonProperty("table") PredictionSource table) { + this.table = table; + } + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter("table") + public PredictionSource table() { + return table; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java index e8bd656bded0..ff96e8c86752 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java @@ -19,6 +19,7 @@ package org.apache.paimon.rest.responses; import org.apache.paimon.catalog.Prediction; +import org.apache.paimon.catalog.PredictionOutput; import org.apache.paimon.catalog.PredictionSource; import org.apache.paimon.catalog.PredictionStatus; import org.apache.paimon.rest.RESTResponse; @@ -42,7 +43,7 @@ public class GetPredictionResponse implements RESTResponse { private final long snapshotId; private final String model; private final String modelVersion; - private final String outputTable; + private final @Nullable PredictionOutput output; private final @Nullable Long startTime; private final @Nullable Long endTime; @@ -55,7 +56,7 @@ public GetPredictionResponse( @JsonProperty("snapshotId") long snapshotId, @JsonProperty("model") String model, @JsonProperty("modelVersion") String modelVersion, - @JsonProperty("outputTable") String outputTable, + @Nullable @JsonProperty("output") PredictionOutput output, @Nullable @JsonProperty("startTime") Long startTime, @Nullable @JsonProperty("endTime") Long endTime) { this.predictionId = predictionId; @@ -65,7 +66,7 @@ public GetPredictionResponse( this.snapshotId = snapshotId; this.model = model; this.modelVersion = modelVersion; - this.outputTable = outputTable; + this.output = output; this.startTime = startTime; this.endTime = endTime; } @@ -79,7 +80,7 @@ public GetPredictionResponse(Prediction prediction) { prediction.snapshotId(), prediction.model(), prediction.modelVersion(), - prediction.outputTable(), + prediction.output(), prediction.startTime(), prediction.endTime()); } @@ -121,9 +122,11 @@ public String getModelVersion() { return modelVersion; } - @JsonGetter("outputTable") - public String getOutputTable() { - return outputTable; + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter("output") + public PredictionOutput getOutput() { + return output; } @Nullable @@ -149,7 +152,7 @@ public Prediction toPrediction() { snapshotId, model, modelVersion, - outputTable, + output, startTime, endTime); } diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java index 4256b990e4a4..4cfdcb825367 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java @@ -20,6 +20,7 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.catalog.Prediction; +import org.apache.paimon.catalog.PredictionOutput; import org.apache.paimon.catalog.PredictionSource; import org.apache.paimon.catalog.PredictionSpec; import org.apache.paimon.catalog.PredictionStatus; @@ -65,7 +66,14 @@ void testRequestAndResponseWireFormat() throws Exception { assertThat(decoded.toPrediction().status().terminationDetails()).isNull(); assertThat(decoded.getName()).isNull(); assertThat(RESTApi.fromJson(RESTApi.toJson(response), Map.class)) - .doesNotContainKeys("createdAt", "startedAt", "finishedAt", "startTime", "endTime"); + .doesNotContainKeys( + "createdAt", + "startedAt", + "finishedAt", + "outputTable", + "output", + "startTime", + "endTime"); GetPredictionResponse completed = new GetPredictionResponse( @@ -80,13 +88,19 @@ void testRequestAndResponseWireFormat() throws Exception { 7, "quality.qwen_vl", "1", - "system.prediction_results", + new PredictionOutput( + new PredictionSource("system", "prediction_results")), 10L, 20L)); - assertThat(RESTApi.fromJson(RESTApi.toJson(completed), Map.class)) + Map completedMap = RESTApi.fromJson(RESTApi.toJson(completed), Map.class); + assertThat(completedMap) .containsEntry("startTime", 10) .containsEntry("endTime", 20) - .doesNotContainKeys("createdAt", "startedAt", "finishedAt"); + .doesNotContainKeys("createdAt", "startedAt", "finishedAt", "outputTable"); + Map output = (Map) completedMap.get("output"); + assertThat((Map) output.get("table")) + .containsEntry("database", "system") + .containsEntry("table", "prediction_results"); ListPredictionsResponse empty = RESTApi.fromJson("{}", ListPredictionsResponse.class); assertThat(empty.getPredictions()).isEmpty(); @@ -180,7 +194,7 @@ private static Prediction prediction() { 7, "quality.qwen_vl", "1", - "system.prediction_results", + null, null, null); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java index d9e5e2018068..785ce58ba520 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java @@ -138,7 +138,7 @@ private static String predictionJson(String status) { + "\"table\":\"clips\"},\"name\":\"daily-quality\",\"status\":{\"state\":\"" + status + "\"},\"snapshotId\":7,\"model\":\"quality.qwen_vl\"," - + "\"modelVersion\":\"1\",\"outputTable\":\"system.results\"}"; + + "\"modelVersion\":\"1\"}"; } private static String terminatedPredictionJson(String code) { @@ -147,7 +147,8 @@ private static String terminatedPredictionJson(String code) { + "\"status\":{\"state\":\"TERMINATED\",\"terminationDetails\":{\"code\":\"" + code + "\"}},\"snapshotId\":7,\"model\":\"quality.qwen_vl\"," - + "\"modelVersion\":\"1\",\"outputTable\":\"system.results\"," + + "\"modelVersion\":\"1\",\"output\":{\"table\":{" + + "\"database\":\"system\",\"table\":\"results\"}}," + "\"startTime\":10,\"endTime\":20}"; } diff --git a/paimon-python/pypaimon/api/prediction.py b/paimon-python/pypaimon/api/prediction.py index 8cd545f46ba6..da6f76e56ea3 100644 --- a/paimon-python/pypaimon/api/prediction.py +++ b/paimon-python/pypaimon/api/prediction.py @@ -23,6 +23,7 @@ from pypaimon.api.api_response import PagedResponse, RESTResponse from pypaimon.catalog.prediction import ( Prediction, + PredictionOutput as CatalogPredictionOutput, PredictionSpec, PredictionStatus, _copy_string_list, @@ -51,6 +52,31 @@ def to_identifier(self) -> Identifier: return Identifier.create(self.database, self.table) +@dataclass +class PredictionOutput: + table: Optional[PredictionSource] = optional_json_field( + "table", "non_null") + + def __post_init__(self): + if self.table is not None and not isinstance( + self.table, PredictionSource): + raise ValueError("table must be a PredictionSource") + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "PredictionOutput": + table = data.get("table") + return cls( + table=(PredictionSource(**table) + if isinstance(table, dict) else table), + ) + + def to_catalog_output(self) -> CatalogPredictionOutput: + return CatalogPredictionOutput( + table=(self.table.to_identifier() + if self.table is not None else None), + ) + + @dataclass class SubmitPredictionRequest(RESTRequest): source: PredictionSource @@ -132,8 +158,9 @@ class GetPredictionResponse(RESTResponse): snapshot_id: int = json_field("snapshotId", default=0) model: str = "" model_version: str = json_field("modelVersion", default="") - output_table: str = json_field("outputTable", default="") name: Optional[str] = optional_json_field("name", "non_null") + output: Optional[PredictionOutput] = optional_json_field( + "output", "non_null") start_time: Optional[int] = optional_json_field("startTime", "non_null") end_time: Optional[int] = optional_json_field("endTime", "non_null") @@ -151,7 +178,9 @@ def from_dict(cls, data: Dict[str, Any]) -> "GetPredictionResponse": snapshot_id=data.get("snapshotId"), model=data.get("model"), model_version=data.get("modelVersion"), - output_table=data.get("outputTable"), + output=(PredictionOutput.from_dict(data.get("output")) + if isinstance(data.get("output"), dict) + else data.get("output")), start_time=data.get("startTime"), end_time=data.get("endTime"), ) @@ -165,7 +194,8 @@ def to_prediction(self) -> Prediction: snapshot_id=self.snapshot_id, model=self.model, model_version=self.model_version, - output_table=self.output_table, + output=(self.output.to_catalog_output() + if self.output is not None else None), start_time=self.start_time, end_time=self.end_time, ) diff --git a/paimon-python/pypaimon/catalog/prediction.py b/paimon-python/pypaimon/catalog/prediction.py index e9aae7df5acf..9fc0bbb5ec31 100644 --- a/paimon-python/pypaimon/catalog/prediction.py +++ b/paimon-python/pypaimon/catalog/prediction.py @@ -143,6 +143,17 @@ def __post_init__(self): self.options = dict(self.options) +@dataclass +class PredictionOutput: + """Known output locations of an asynchronous prediction.""" + + table: Optional[Identifier] = None + + def __post_init__(self): + if self.table is not None and not isinstance(self.table, Identifier): + raise ValueError("table must be an Identifier") + + @dataclass class Prediction: """Durable metadata for one asynchronous prediction job.""" @@ -153,8 +164,8 @@ class Prediction: snapshot_id: int model: str model_version: str - output_table: str name: Optional[str] = None + output: Optional[PredictionOutput] = None start_time: Optional[int] = None end_time: Optional[int] = None @@ -168,7 +179,9 @@ def __post_init__(self): raise ValueError("snapshot_id must be a positive integer") _nonblank(self.model, "model") _nonblank(self.model_version, "model_version") - _nonblank(self.output_table, "output_table") + if self.output is not None and not isinstance( + self.output, PredictionOutput): + raise ValueError("output must be a PredictionOutput") for name in ("start_time", "end_time"): value = getattr(self, name) if value is not None and (type(value) is not int or value < 0): diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index d16e2d84491b..0733deb1ddfa 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -25,12 +25,14 @@ from pypaimon.api.prediction import ( GetPredictionResponse, ListPredictionsResponse, + PredictionOutput as ApiPredictionOutput, PredictionSource, SubmitPredictionRequest, ) from pypaimon.api.rest_api import RESTApi from pypaimon.catalog.prediction import ( Prediction, + PredictionOutput, PredictionState, PredictionSpec, PredictionStatus, @@ -70,7 +72,6 @@ def _prediction(status=None, **changes): snapshot_id=7, model="quality.qwen_vl", model_version="1", - output_table="system.prediction_results", ) values.update(changes) return Prediction(**values) @@ -87,7 +88,6 @@ def _response(status=None, **changes): snapshot_id=7, model="quality.qwen_vl", model_version="1", - output_table="system.prediction_results", ) values.update(changes) return GetPredictionResponse(**values) @@ -119,7 +119,7 @@ def test_request_and_response_wire_format(self): pending_body = json.loads(JSON.to_json(_response())) for field in ( "createdAt", "startedAt", "finishedAt", - "startTime", "endTime"): + "outputTable", "output", "startTime", "endTime"): self.assertNotIn(field, pending_body) terminated = PredictionStatus( PredictionState.TERMINATED, @@ -128,8 +128,14 @@ def test_request_and_response_wire_format(self): "Model request failed", ), ) - terminated_body = json.loads(JSON.to_json(_response( - terminated, start_time=10, end_time=20))) + completed_response = _response( + terminated, + output=ApiPredictionOutput( + PredictionSource("system", "prediction_results")), + start_time=10, + end_time=20, + ) + terminated_body = json.loads(JSON.to_json(completed_response)) self.assertEqual( { "state": "TERMINATED", @@ -142,14 +148,22 @@ def test_request_and_response_wire_format(self): ) self.assertEqual(10, terminated_body["startTime"]) self.assertEqual(20, terminated_body["endTime"]) - for field in ("createdAt", "startedAt", "finishedAt"): + self.assertEqual( + {"table": {"database": "system", "table": "prediction_results"}}, + terminated_body["output"], + ) + for field in ("createdAt", "startedAt", "finishedAt", "outputTable"): self.assertNotIn(field, terminated_body) + completed = JSON.from_json( + JSON.to_json(completed_response), GetPredictionResponse) self.assertEqual( terminated, - JSON.from_json( - JSON.to_json(_response(terminated)), - GetPredictionResponse, - ).status, + completed.status, + ) + self.assertEqual( + PredictionOutput( + Identifier.create("system", "prediction_results")), + completed.to_prediction().output, ) page = JSON.from_json( JSON.to_json(ListPredictionsResponse([_response()], "next")), From f1576bb5af128a7ceaeba535b75a8849f0f62886 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Mon, 16 Nov 2026 21:16:06 +0800 Subject: [PATCH 09/41] [core][python] Remove unversioned prediction model field --- docs/static/rest-catalog-open-api.yaml | 9 ++++----- .../org/apache/paimon/catalog/Prediction.java | 8 -------- .../rest/responses/GetPredictionResponse.java | 20 +------------------ .../paimon/rest/RESTApiPredictionTest.java | 6 +++--- .../rest/RESTCatalogPredictionTest.java | 5 ++--- paimon-python/pypaimon/api/prediction.py | 3 --- paimon-python/pypaimon/catalog/prediction.py | 2 -- .../pypaimon/tests/rest/prediction_test.py | 9 +++++---- 8 files changed, 15 insertions(+), 47 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 22668929eeff..650e5bd8be38 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -139,9 +139,9 @@ paths: summary: Submit a prediction job operationId: submitPrediction description: >- - Accepts a job after fixing the source snapshot and model version. Reusing an - idempotencyKey with the same payload returns the existing job; conflicting - reuse returns 409. + Accepts a job after fixing the source snapshot and resolving the registered model. + Reusing an idempotencyKey with the same payload returns the existing job; + conflicting reuse returns 409. parameters: - {name: prefix, in: path, required: true, schema: {type: string}} requestBody: @@ -4765,7 +4765,7 @@ components: GetPredictionResponse: type: object additionalProperties: false - required: [predictionId, source, status, snapshotId, model, modelVersion] + required: [predictionId, source, status, snapshotId, model] properties: predictionId: {type: string, minLength: 1} source: @@ -4775,7 +4775,6 @@ components: $ref: '#/components/schemas/PredictionStatus' snapshotId: {type: integer, format: int64, minimum: 1} model: {type: string, minLength: 1} - modelVersion: {type: string, minLength: 1} output: $ref: '#/components/schemas/PredictionOutput' startTime: {type: integer, format: int64, minimum: 0} diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java index c826f8082b45..569f6f5922f1 100644 --- a/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java @@ -34,7 +34,6 @@ public class Prediction { private final PredictionStatus status; private final long snapshotId; private final String model; - private final String modelVersion; private final @Nullable PredictionOutput output; private final @Nullable Long startTime; private final @Nullable Long endTime; @@ -46,7 +45,6 @@ public Prediction( PredictionStatus status, long snapshotId, String model, - String modelVersion, @Nullable PredictionOutput output, @Nullable Long startTime, @Nullable Long endTime) { @@ -55,7 +53,6 @@ public Prediction( checkArgument(status != null, "status must not be null"); checkArgument(snapshotId > 0, "snapshotId must be positive"); checkNonBlank(model, "model"); - checkNonBlank(modelVersion, "modelVersion"); checkOptionalNonNegative(startTime, "startTime"); checkOptionalNonNegative(endTime, "endTime"); this.predictionId = predictionId; @@ -64,7 +61,6 @@ public Prediction( this.status = status; this.snapshotId = snapshotId; this.model = model; - this.modelVersion = modelVersion; this.output = output; this.startTime = startTime; this.endTime = endTime; @@ -95,10 +91,6 @@ public String model() { return model; } - public String modelVersion() { - return modelVersion; - } - @Nullable public PredictionOutput output() { return output; diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java index ff96e8c86752..f07b8c6947b8 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java @@ -42,7 +42,6 @@ public class GetPredictionResponse implements RESTResponse { private final PredictionStatus status; private final long snapshotId; private final String model; - private final String modelVersion; private final @Nullable PredictionOutput output; private final @Nullable Long startTime; private final @Nullable Long endTime; @@ -55,7 +54,6 @@ public GetPredictionResponse( @JsonProperty("status") PredictionStatus status, @JsonProperty("snapshotId") long snapshotId, @JsonProperty("model") String model, - @JsonProperty("modelVersion") String modelVersion, @Nullable @JsonProperty("output") PredictionOutput output, @Nullable @JsonProperty("startTime") Long startTime, @Nullable @JsonProperty("endTime") Long endTime) { @@ -65,7 +63,6 @@ public GetPredictionResponse( this.status = status; this.snapshotId = snapshotId; this.model = model; - this.modelVersion = modelVersion; this.output = output; this.startTime = startTime; this.endTime = endTime; @@ -79,7 +76,6 @@ public GetPredictionResponse(Prediction prediction) { prediction.status(), prediction.snapshotId(), prediction.model(), - prediction.modelVersion(), prediction.output(), prediction.startTime(), prediction.endTime()); @@ -117,11 +113,6 @@ public String getModel() { return model; } - @JsonGetter("modelVersion") - public String getModelVersion() { - return modelVersion; - } - @Nullable @JsonInclude(JsonInclude.Include.NON_NULL) @JsonGetter("output") @@ -145,15 +136,6 @@ public Long getEndTime() { public Prediction toPrediction() { return new Prediction( - predictionId, - source, - name, - status, - snapshotId, - model, - modelVersion, - output, - startTime, - endTime); + predictionId, source, name, status, snapshotId, model, output, startTime, endTime); } } diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java index 4cfdcb825367..80aed137fff9 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java @@ -70,6 +70,7 @@ void testRequestAndResponseWireFormat() throws Exception { "createdAt", "startedAt", "finishedAt", + "modelVersion", "outputTable", "output", "startTime", @@ -87,7 +88,6 @@ void testRequestAndResponseWireFormat() throws Exception { PredictionStatus.TerminationCode.SUCCESS, null)), 7, "quality.qwen_vl", - "1", new PredictionOutput( new PredictionSource("system", "prediction_results")), 10L, @@ -96,7 +96,8 @@ void testRequestAndResponseWireFormat() throws Exception { assertThat(completedMap) .containsEntry("startTime", 10) .containsEntry("endTime", 20) - .doesNotContainKeys("createdAt", "startedAt", "finishedAt", "outputTable"); + .doesNotContainKeys( + "createdAt", "startedAt", "finishedAt", "modelVersion", "outputTable"); Map output = (Map) completedMap.get("output"); assertThat((Map) output.get("table")) .containsEntry("database", "system") @@ -193,7 +194,6 @@ private static Prediction prediction() { new PredictionStatus(PredictionStatus.State.PENDING, null), 7, "quality.qwen_vl", - "1", null, null, null); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java index 785ce58ba520..cc4a524b35ec 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java @@ -137,8 +137,7 @@ private static String predictionJson(String status) { return "{\"predictionId\":\"prediction-1\",\"source\":{\"database\":\"quality\"," + "\"table\":\"clips\"},\"name\":\"daily-quality\",\"status\":{\"state\":\"" + status - + "\"},\"snapshotId\":7,\"model\":\"quality.qwen_vl\"," - + "\"modelVersion\":\"1\"}"; + + "\"},\"snapshotId\":7,\"model\":\"quality.qwen_vl\"}"; } private static String terminatedPredictionJson(String code) { @@ -147,7 +146,7 @@ private static String terminatedPredictionJson(String code) { + "\"status\":{\"state\":\"TERMINATED\",\"terminationDetails\":{\"code\":\"" + code + "\"}},\"snapshotId\":7,\"model\":\"quality.qwen_vl\"," - + "\"modelVersion\":\"1\",\"output\":{\"table\":{" + + "\"output\":{\"table\":{" + "\"database\":\"system\",\"table\":\"results\"}}," + "\"startTime\":10,\"endTime\":20}"; } diff --git a/paimon-python/pypaimon/api/prediction.py b/paimon-python/pypaimon/api/prediction.py index da6f76e56ea3..3469e9fd50ea 100644 --- a/paimon-python/pypaimon/api/prediction.py +++ b/paimon-python/pypaimon/api/prediction.py @@ -157,7 +157,6 @@ class GetPredictionResponse(RESTResponse): status: PredictionStatus = None snapshot_id: int = json_field("snapshotId", default=0) model: str = "" - model_version: str = json_field("modelVersion", default="") name: Optional[str] = optional_json_field("name", "non_null") output: Optional[PredictionOutput] = optional_json_field( "output", "non_null") @@ -177,7 +176,6 @@ def from_dict(cls, data: Dict[str, Any]) -> "GetPredictionResponse": status=PredictionStatus.from_dict(data.get("status")), snapshot_id=data.get("snapshotId"), model=data.get("model"), - model_version=data.get("modelVersion"), output=(PredictionOutput.from_dict(data.get("output")) if isinstance(data.get("output"), dict) else data.get("output")), @@ -193,7 +191,6 @@ def to_prediction(self) -> Prediction: status=self.status, snapshot_id=self.snapshot_id, model=self.model, - model_version=self.model_version, output=(self.output.to_catalog_output() if self.output is not None else None), start_time=self.start_time, diff --git a/paimon-python/pypaimon/catalog/prediction.py b/paimon-python/pypaimon/catalog/prediction.py index 9fc0bbb5ec31..fa3c1a1d4cee 100644 --- a/paimon-python/pypaimon/catalog/prediction.py +++ b/paimon-python/pypaimon/catalog/prediction.py @@ -163,7 +163,6 @@ class Prediction: status: PredictionStatus snapshot_id: int model: str - model_version: str name: Optional[str] = None output: Optional[PredictionOutput] = None start_time: Optional[int] = None @@ -178,7 +177,6 @@ def __post_init__(self): if type(self.snapshot_id) is not int or self.snapshot_id <= 0: raise ValueError("snapshot_id must be a positive integer") _nonblank(self.model, "model") - _nonblank(self.model_version, "model_version") if self.output is not None and not isinstance( self.output, PredictionOutput): raise ValueError("output must be a PredictionOutput") diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index 0733deb1ddfa..c74b8b1ef4db 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -71,7 +71,6 @@ def _prediction(status=None, **changes): status=status, snapshot_id=7, model="quality.qwen_vl", - model_version="1", ) values.update(changes) return Prediction(**values) @@ -87,7 +86,6 @@ def _response(status=None, **changes): status=status, snapshot_id=7, model="quality.qwen_vl", - model_version="1", ) values.update(changes) return GetPredictionResponse(**values) @@ -119,7 +117,8 @@ def test_request_and_response_wire_format(self): pending_body = json.loads(JSON.to_json(_response())) for field in ( "createdAt", "startedAt", "finishedAt", - "outputTable", "output", "startTime", "endTime"): + "modelVersion", "outputTable", "output", "startTime", + "endTime"): self.assertNotIn(field, pending_body) terminated = PredictionStatus( PredictionState.TERMINATED, @@ -152,7 +151,9 @@ def test_request_and_response_wire_format(self): {"table": {"database": "system", "table": "prediction_results"}}, terminated_body["output"], ) - for field in ("createdAt", "startedAt", "finishedAt", "outputTable"): + for field in ( + "createdAt", "startedAt", "finishedAt", "modelVersion", + "outputTable"): self.assertNotIn(field, terminated_body) completed = JSON.from_json( JSON.to_json(completed_response), GetPredictionResponse) From 465525203a1d56b8826dd90b9b9875551d71295f Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Mon, 16 Nov 2026 21:34:34 +0800 Subject: [PATCH 10/41] [core][python] Infer prediction source identity --- docs/static/rest-catalog-open-api.yaml | 8 ++++++-- .../apache/paimon/catalog/PredictionSpec.java | 14 ++++++++----- .../requests/SubmitPredictionRequest.java | 15 +++++++------- .../paimon/rest/RESTApiPredictionTest.java | 9 +++++---- .../rest/RESTCatalogPredictionTest.java | 3 ++- paimon-python/pypaimon/api/prediction.py | 10 ++++++---- paimon-python/pypaimon/catalog/prediction.py | 6 +++--- paimon-python/pypaimon/multimodal/table.py | 5 +++-- .../pypaimon/tests/rest/prediction_test.py | 20 +++++++++++++------ .../tests/rest/rest_api_coverage_test.py | 1 - 10 files changed, 56 insertions(+), 35 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 650e5bd8be38..0417c8837e3e 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -4711,7 +4711,7 @@ components: SubmitPredictionRequest: type: object additionalProperties: false - required: [source, idempotencyKey, model, inputs, resultColumns, prompt, outputSchema] + required: [source, idempotencyKey, model, inputs, prompt, outputSchema] properties: source: $ref: '#/components/schemas/PredictionSource' @@ -4734,10 +4734,14 @@ components: type: string minLength: 1 description: Model input-to-column mappings. - resultColumns: + mergeKeyColumns: type: array uniqueItems: true items: {type: string, minLength: 1} + description: >- + Stable source columns used when table primary keys or row IDs are unavailable. + Omit to let the server derive identity; without stable identity the prediction + runs as a full refresh. prompt: type: string minLength: 1 diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java index 91252778dfb7..f6be04da5a91 100644 --- a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java @@ -37,7 +37,7 @@ public class PredictionSpec { private final @Nullable String name; private final String model; private final Map inputs; - private final List resultColumns; + private final List mergeKeyColumns; private final String prompt; private final Map outputSchema; private final @Nullable String where; @@ -49,7 +49,7 @@ public PredictionSpec( @Nullable String name, String model, Map inputs, - List resultColumns, + @Nullable List mergeKeyColumns, String prompt, Map outputSchema, @Nullable String where, @@ -68,7 +68,7 @@ public PredictionSpec( this.name = name; this.model = model; this.inputs = copyStringMap(inputs, "inputs", true); - this.resultColumns = copyStringList(resultColumns, "resultColumns"); + this.mergeKeyColumns = copyOptionalStringList(mergeKeyColumns, "mergeKeyColumns"); this.prompt = prompt; this.outputSchema = copyStringMap(outputSchema, "outputSchema", true); this.where = where; @@ -93,8 +93,8 @@ public Map inputs() { return inputs; } - public List resultColumns() { - return resultColumns; + public List mergeKeyColumns() { + return mergeKeyColumns; } public String prompt() { @@ -149,6 +149,10 @@ private static List copyStringList(List values, String name) { return Collections.unmodifiableList(copy); } + private static List copyOptionalStringList(@Nullable List values, String name) { + return values == null ? Collections.emptyList() : copyStringList(values, name); + } + private static void checkOptionalNonBlank(@Nullable String value, String name) { if (value != null) { checkNonBlank(value, name); diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java index 3ee685966b15..db4de820c52a 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java @@ -46,7 +46,7 @@ public class SubmitPredictionRequest implements RESTRequest { private static final String FIELD_IDEMPOTENCY_KEY = "idempotencyKey"; private static final String FIELD_MODEL = "model"; private static final String FIELD_INPUTS = "inputs"; - private static final String FIELD_RESULT_COLUMNS = "resultColumns"; + private static final String FIELD_MERGE_KEY_COLUMNS = "mergeKeyColumns"; private static final String FIELD_PROMPT = "prompt"; private static final String FIELD_OUTPUT_SCHEMA = "outputSchema"; private static final String FIELD_WHERE = "where"; @@ -65,7 +65,7 @@ public SubmitPredictionRequest( @Nullable @JsonProperty(FIELD_IDEMPOTENCY_KEY) String idempotencyKey, @JsonProperty(FIELD_MODEL) String model, @JsonProperty(FIELD_INPUTS) Map inputs, - @JsonProperty(FIELD_RESULT_COLUMNS) List resultColumns, + @Nullable @JsonProperty(FIELD_MERGE_KEY_COLUMNS) List mergeKeyColumns, @JsonProperty(FIELD_PROMPT) String prompt, @JsonProperty(FIELD_OUTPUT_SCHEMA) Map outputSchema, @Nullable @JsonProperty(FIELD_WHERE) String where, @@ -84,7 +84,7 @@ public SubmitPredictionRequest( name, model, inputs, - resultColumns, + mergeKeyColumns, prompt, outputSchema, where, @@ -100,7 +100,7 @@ public SubmitPredictionRequest(Identifier source, PredictionSpec spec) { null, spec.model(), spec.inputs(), - spec.resultColumns(), + spec.mergeKeyColumns(), spec.prompt(), spec.outputSchema(), spec.where(), @@ -136,9 +136,10 @@ public Map getInputs() { return spec.inputs(); } - @JsonGetter(FIELD_RESULT_COLUMNS) - public List getResultColumns() { - return spec.resultColumns(); + @JsonInclude(JsonInclude.Include.NON_EMPTY) + @JsonGetter(FIELD_MERGE_KEY_COLUMNS) + public List getMergeKeyColumns() { + return spec.mergeKeyColumns(); } @JsonGetter(FIELD_PROMPT) diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java index 80aed137fff9..6695756026f9 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java @@ -52,8 +52,9 @@ void testRequestAndResponseWireFormat() throws Exception { .isEqualTo(requestMap.get("idempotencyKey")); assertThat(requestMap) .containsEntry("model", "quality.qwen_vl") - .doesNotContainKeys("name", "where", "snapshotId", "tagName"); - assertThat(requestMap.get("resultColumns")).isEqualTo(Collections.singletonList("clip_id")); + .doesNotContainKeys("name", "where", "snapshotId", "tagName", "resultColumns"); + assertThat(requestMap.get("mergeKeyColumns")) + .isEqualTo(Collections.singletonList("clip_id")); assertThat((Map) requestMap.get("source")) .containsEntry("database", "quality") .containsEntry("table", "clips"); @@ -118,7 +119,7 @@ void testSpecValidation() { null, "quality.qwen_vl", Collections.emptyMap(), - Collections.emptyList(), + null, "Classify", Collections.singletonMap("result", "STRING"), null, @@ -142,7 +143,7 @@ void testSpecValidation() { null, Collections.emptyMap())) .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("resultColumns must not contain duplicates"); + .hasMessageContaining("mergeKeyColumns must not contain duplicates"); } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java index cc4a524b35ec..58f078e9433c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java @@ -57,6 +57,7 @@ void testPredictionLifecycleUsesSharedRestContract() throws Exception { Map request = RESTApi.fromJson(submit.getBody().readUtf8(), Map.class); assertThat(request.get("idempotencyKey")).isInstanceOf(String.class); assertThat((String) request.get("idempotencyKey")).isNotBlank(); + assertThat(request).doesNotContainKeys("resultColumns", "mergeKeyColumns"); assertThat((Map) request.get("source")) .containsEntry("database", "quality") .containsEntry("table", "clips"); @@ -115,7 +116,7 @@ private static PredictionSpec spec() { "daily-quality", "quality.qwen_vl", Collections.singletonMap("video", "content"), - Collections.singletonList("clip_id"), + null, "Classify", Collections.singletonMap("result", "STRING"), null, diff --git a/paimon-python/pypaimon/api/prediction.py b/paimon-python/pypaimon/api/prediction.py index 3469e9fd50ea..01775232770a 100644 --- a/paimon-python/pypaimon/api/prediction.py +++ b/paimon-python/pypaimon/api/prediction.py @@ -82,9 +82,10 @@ class SubmitPredictionRequest(RESTRequest): source: PredictionSource model: str inputs: Dict[str, str] - result_columns: List[str] = json_field("resultColumns") prompt: str output_schema: Dict[str, str] = json_field("outputSchema") + merge_key_columns: Optional[List[str]] = optional_json_field( + "mergeKeyColumns", "non_null") name: Optional[str] = optional_json_field("name", "non_null") idempotency_key: Optional[str] = optional_json_field( "idempotencyKey", "non_null") @@ -116,8 +117,9 @@ def __post_init__(self): if self.snapshot_id is not None and self.tag_name is not None: raise ValueError("snapshot_id and tag_name are mutually exclusive") self.inputs = _copy_string_mapping(self.inputs, "inputs", True) - self.result_columns = _copy_string_list( - self.result_columns, "result_columns") + if self.merge_key_columns is not None: + self.merge_key_columns = _copy_string_list( + self.merge_key_columns, "merge_key_columns") self.output_schema = _copy_string_mapping( self.output_schema, "output_schema", True) if not isinstance(self.options, dict): @@ -133,9 +135,9 @@ def from_spec( name=spec.name, model=spec.model, inputs=spec.inputs, - result_columns=spec.result_columns, prompt=spec.prompt, output_schema=spec.output_schema, + merge_key_columns=spec.merge_key_columns or None, where=spec.where, snapshot_id=spec.snapshot_id, tag_name=spec.tag_name, diff --git a/paimon-python/pypaimon/catalog/prediction.py b/paimon-python/pypaimon/catalog/prediction.py index fa3c1a1d4cee..1913d0001b30 100644 --- a/paimon-python/pypaimon/catalog/prediction.py +++ b/paimon-python/pypaimon/catalog/prediction.py @@ -109,9 +109,9 @@ class PredictionSpec: model: str inputs: Dict[str, str] - result_columns: List[str] prompt: str output_schema: Dict[str, str] + merge_key_columns: List[str] = field(default_factory=list) name: Optional[str] = None where: Optional[str] = None snapshot_id: Optional[int] = None @@ -134,8 +134,8 @@ def __post_init__(self): if self.snapshot_id is not None and self.tag_name is not None: raise ValueError("snapshot_id and tag_name are mutually exclusive") self.inputs = _copy_string_mapping(self.inputs, "inputs", True) - self.result_columns = _copy_string_list( - self.result_columns, "result_columns") + self.merge_key_columns = _copy_string_list( + self.merge_key_columns, "merge_key_columns") self.output_schema = _copy_string_mapping( self.output_schema, "output_schema", True) if not isinstance(self.options, dict): diff --git a/paimon-python/pypaimon/multimodal/table.py b/paimon-python/pypaimon/multimodal/table.py index c872a374150e..741d38aeadfe 100644 --- a/paimon-python/pypaimon/multimodal/table.py +++ b/paimon-python/pypaimon/multimodal/table.py @@ -116,9 +116,9 @@ def predict( *, model, inputs, - result_columns, prompt, output_schema, + merge_key_columns=None, name=None, where=None, options=None): @@ -144,9 +144,10 @@ def predict( name=name, model=model, inputs=inputs, - result_columns=result_columns, prompt=prompt, output_schema=output_schema, + merge_key_columns=( + [] if merge_key_columns is None else merge_key_columns), where=where, snapshot_id=self.raw_table.options.scan_snapshot_id(), tag_name=self.raw_table.options.scan_tag_name(), diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index c74b8b1ef4db..c5fdb77741c7 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -53,7 +53,6 @@ def _spec(**changes): values = dict( model="quality.qwen_vl", inputs={"video": "content"}, - result_columns=["clip_id"], prompt="Classify", output_schema={"result": "STRING"}, ) @@ -102,12 +101,20 @@ def test_request_and_response_wire_format(self): body["idempotencyKey"], json.loads(JSON.to_json(request))["idempotencyKey"], ) - self.assertEqual(["clip_id"], body["resultColumns"]) + self.assertNotIn("resultColumns", body) + self.assertNotIn("mergeKeyColumns", body) self.assertEqual({"result": "STRING"}, body["outputSchema"]) self.assertEqual( {"database": "quality", "table": "clips"}, body["source"]) self.assertNotIn("snapshotId", body) + keyed = SubmitPredictionRequest.from_spec( + Identifier.create("quality", "clips"), + _spec(merge_key_columns=["clip_id"]), + ) + keyed_body = json.loads(JSON.to_json(keyed)) + self.assertEqual(["clip_id"], keyed_body["mergeKeyColumns"]) + response = JSON.from_json( JSON.to_json(_response()), GetPredictionResponse) self.assertEqual( @@ -178,8 +185,8 @@ def test_validation(self): dict(model=""), dict(inputs={}), dict(inputs={"video": ""}), - dict(result_columns=["id", "id"]), - dict(result_columns="id"), + dict(merge_key_columns=["id", "id"]), + dict(merge_key_columns="id"), dict(prompt=" "), dict(output_schema={}), dict(snapshot_id=0), @@ -209,16 +216,16 @@ def test_table_submit_fixes_view_and_returns_job(self): name="daily-quality", model="quality.qwen_vl", inputs={"video": "content"}, - result_columns=["clip_id"], prompt="Classify", output_schema={"result": "STRING"}, ) self.assertIsInstance(job, PredictionJob) self.assertEqual("prediction-1", job.prediction_id) - sent = catalog.submit_prediction.call_args.args[1] + sent = catalog.submit_prediction.call_args[0][1] self.assertEqual(7, sent.snapshot_id) self.assertIsNone(sent.tag_name) + self.assertEqual([], sent.merge_key_columns) raw.new_read_builder.assert_not_called() def test_wait_is_client_side_polling(self): @@ -298,6 +305,7 @@ def do_GET(self): calls[-1][2]["source"], ) self.assertTrue(calls[-1][2]["idempotencyKey"]) + self.assertNotIn("mergeKeyColumns", calls[-1][2]) page = catalog.list_predictions( identifier, diff --git a/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py b/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py index bfa1999c14f6..b589db3fb93f 100644 --- a/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py +++ b/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py @@ -68,7 +68,6 @@ source=PredictionSource("db", "t"), model="db.qwen_vl", inputs={"video": "content"}, - result_columns=["id"], prompt="classify", output_schema={"result": "STRING"}, ), From a3c65bacb39838c7271b753dcdf7546b099434a1 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Mon, 16 Nov 2026 21:44:14 +0800 Subject: [PATCH 11/41] [core][python] Rename prediction prompt to instructions --- docs/static/rest-catalog-open-api.yaml | 5 +++-- .../org/apache/paimon/catalog/PredictionSpec.java | 12 ++++++------ .../rest/requests/SubmitPredictionRequest.java | 14 +++++++------- .../apache/paimon/rest/RESTApiPredictionTest.java | 4 +++- paimon-python/pypaimon/api/prediction.py | 6 +++--- paimon-python/pypaimon/catalog/prediction.py | 4 ++-- paimon-python/pypaimon/multimodal/table.py | 4 ++-- .../pypaimon/tests/rest/prediction_test.py | 8 +++++--- .../pypaimon/tests/rest/rest_api_coverage_test.py | 2 +- 9 files changed, 32 insertions(+), 27 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 0417c8837e3e..27e44311c5e8 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -4711,7 +4711,7 @@ components: SubmitPredictionRequest: type: object additionalProperties: false - required: [source, idempotencyKey, model, inputs, prompt, outputSchema] + required: [source, idempotencyKey, model, inputs, instructions, outputSchema] properties: source: $ref: '#/components/schemas/PredictionSource' @@ -4742,9 +4742,10 @@ components: Stable source columns used when table primary keys or row IDs are unavailable. Omit to let the server derive identity; without stable identity the prediction runs as a full refresh. - prompt: + instructions: type: string minLength: 1 + description: Natural-language guidance for the prediction. outputSchema: type: object minProperties: 1 diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java index f6be04da5a91..e10ef1330a6a 100644 --- a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java @@ -38,7 +38,7 @@ public class PredictionSpec { private final String model; private final Map inputs; private final List mergeKeyColumns; - private final String prompt; + private final String instructions; private final Map outputSchema; private final @Nullable String where; private final @Nullable Long snapshotId; @@ -50,7 +50,7 @@ public PredictionSpec( String model, Map inputs, @Nullable List mergeKeyColumns, - String prompt, + String instructions, Map outputSchema, @Nullable String where, @Nullable Long snapshotId, @@ -58,7 +58,7 @@ public PredictionSpec( @Nullable Map options) { checkOptionalNonBlank(name, "name"); checkNonBlank(model, "model"); - checkNonBlank(prompt, "prompt"); + checkNonBlank(instructions, "instructions"); checkOptionalNonBlank(where, "where"); checkOptionalNonBlank(tagName, "tagName"); checkArgument(snapshotId == null || snapshotId > 0, "snapshotId must be positive"); @@ -69,7 +69,7 @@ public PredictionSpec( this.model = model; this.inputs = copyStringMap(inputs, "inputs", true); this.mergeKeyColumns = copyOptionalStringList(mergeKeyColumns, "mergeKeyColumns"); - this.prompt = prompt; + this.instructions = instructions; this.outputSchema = copyStringMap(outputSchema, "outputSchema", true); this.where = where; this.snapshotId = snapshotId; @@ -97,8 +97,8 @@ public List mergeKeyColumns() { return mergeKeyColumns; } - public String prompt() { - return prompt; + public String instructions() { + return instructions; } public Map outputSchema() { diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java index db4de820c52a..eaedb0e1b2aa 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java @@ -47,7 +47,7 @@ public class SubmitPredictionRequest implements RESTRequest { private static final String FIELD_MODEL = "model"; private static final String FIELD_INPUTS = "inputs"; private static final String FIELD_MERGE_KEY_COLUMNS = "mergeKeyColumns"; - private static final String FIELD_PROMPT = "prompt"; + private static final String FIELD_INSTRUCTIONS = "instructions"; private static final String FIELD_OUTPUT_SCHEMA = "outputSchema"; private static final String FIELD_WHERE = "where"; private static final String FIELD_SNAPSHOT_ID = "snapshotId"; @@ -66,7 +66,7 @@ public SubmitPredictionRequest( @JsonProperty(FIELD_MODEL) String model, @JsonProperty(FIELD_INPUTS) Map inputs, @Nullable @JsonProperty(FIELD_MERGE_KEY_COLUMNS) List mergeKeyColumns, - @JsonProperty(FIELD_PROMPT) String prompt, + @JsonProperty(FIELD_INSTRUCTIONS) String instructions, @JsonProperty(FIELD_OUTPUT_SCHEMA) Map outputSchema, @Nullable @JsonProperty(FIELD_WHERE) String where, @Nullable @JsonProperty(FIELD_SNAPSHOT_ID) Long snapshotId, @@ -85,7 +85,7 @@ public SubmitPredictionRequest( model, inputs, mergeKeyColumns, - prompt, + instructions, outputSchema, where, snapshotId, @@ -101,7 +101,7 @@ public SubmitPredictionRequest(Identifier source, PredictionSpec spec) { spec.model(), spec.inputs(), spec.mergeKeyColumns(), - spec.prompt(), + spec.instructions(), spec.outputSchema(), spec.where(), spec.snapshotId(), @@ -142,9 +142,9 @@ public List getMergeKeyColumns() { return spec.mergeKeyColumns(); } - @JsonGetter(FIELD_PROMPT) - public String getPrompt() { - return spec.prompt(); + @JsonGetter(FIELD_INSTRUCTIONS) + public String getInstructions() { + return spec.instructions(); } @JsonGetter(FIELD_OUTPUT_SCHEMA) diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java index 6695756026f9..278eea7b5d4a 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java @@ -52,7 +52,9 @@ void testRequestAndResponseWireFormat() throws Exception { .isEqualTo(requestMap.get("idempotencyKey")); assertThat(requestMap) .containsEntry("model", "quality.qwen_vl") - .doesNotContainKeys("name", "where", "snapshotId", "tagName", "resultColumns"); + .containsEntry("instructions", "Classify") + .doesNotContainKeys( + "name", "prompt", "where", "snapshotId", "tagName", "resultColumns"); assertThat(requestMap.get("mergeKeyColumns")) .isEqualTo(Collections.singletonList("clip_id")); assertThat((Map) requestMap.get("source")) diff --git a/paimon-python/pypaimon/api/prediction.py b/paimon-python/pypaimon/api/prediction.py index 01775232770a..242b6f6f3a69 100644 --- a/paimon-python/pypaimon/api/prediction.py +++ b/paimon-python/pypaimon/api/prediction.py @@ -82,7 +82,7 @@ class SubmitPredictionRequest(RESTRequest): source: PredictionSource model: str inputs: Dict[str, str] - prompt: str + instructions: str output_schema: Dict[str, str] = json_field("outputSchema") merge_key_columns: Optional[List[str]] = optional_json_field( "mergeKeyColumns", "non_null") @@ -97,7 +97,7 @@ class SubmitPredictionRequest(RESTRequest): def __post_init__(self): for value, field_name in ( (self.model, "model"), - (self.prompt, "prompt"), + (self.instructions, "instructions"), ): _nonblank(value, field_name) for value, field_name in ( @@ -135,7 +135,7 @@ def from_spec( name=spec.name, model=spec.model, inputs=spec.inputs, - prompt=spec.prompt, + instructions=spec.instructions, output_schema=spec.output_schema, merge_key_columns=spec.merge_key_columns or None, where=spec.where, diff --git a/paimon-python/pypaimon/catalog/prediction.py b/paimon-python/pypaimon/catalog/prediction.py index 1913d0001b30..45536744a933 100644 --- a/paimon-python/pypaimon/catalog/prediction.py +++ b/paimon-python/pypaimon/catalog/prediction.py @@ -109,7 +109,7 @@ class PredictionSpec: model: str inputs: Dict[str, str] - prompt: str + instructions: str output_schema: Dict[str, str] merge_key_columns: List[str] = field(default_factory=list) name: Optional[str] = None @@ -120,7 +120,7 @@ class PredictionSpec: def __post_init__(self): _nonblank(self.model, "model") - _nonblank(self.prompt, "prompt") + _nonblank(self.instructions, "instructions") for value, name in ( (self.name, "name"), (self.where, "where"), diff --git a/paimon-python/pypaimon/multimodal/table.py b/paimon-python/pypaimon/multimodal/table.py index 741d38aeadfe..f76afd349b62 100644 --- a/paimon-python/pypaimon/multimodal/table.py +++ b/paimon-python/pypaimon/multimodal/table.py @@ -116,7 +116,7 @@ def predict( *, model, inputs, - prompt, + instructions, output_schema, merge_key_columns=None, name=None, @@ -144,7 +144,7 @@ def predict( name=name, model=model, inputs=inputs, - prompt=prompt, + instructions=instructions, output_schema=output_schema, merge_key_columns=( [] if merge_key_columns is None else merge_key_columns), diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index c5fdb77741c7..978d223dfde3 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -53,7 +53,7 @@ def _spec(**changes): values = dict( model="quality.qwen_vl", inputs={"video": "content"}, - prompt="Classify", + instructions="Classify", output_schema={"result": "STRING"}, ) values.update(changes) @@ -103,6 +103,8 @@ def test_request_and_response_wire_format(self): ) self.assertNotIn("resultColumns", body) self.assertNotIn("mergeKeyColumns", body) + self.assertNotIn("prompt", body) + self.assertEqual("Classify", body["instructions"]) self.assertEqual({"result": "STRING"}, body["outputSchema"]) self.assertEqual( {"database": "quality", "table": "clips"}, body["source"]) @@ -187,7 +189,7 @@ def test_validation(self): dict(inputs={"video": ""}), dict(merge_key_columns=["id", "id"]), dict(merge_key_columns="id"), - dict(prompt=" "), + dict(instructions=" "), dict(output_schema={}), dict(snapshot_id=0), dict(snapshot_id=1, tag_name="v1"), @@ -216,7 +218,7 @@ def test_table_submit_fixes_view_and_returns_job(self): name="daily-quality", model="quality.qwen_vl", inputs={"video": "content"}, - prompt="Classify", + instructions="Classify", output_schema={"result": "STRING"}, ) diff --git a/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py b/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py index b589db3fb93f..367d792e99f1 100644 --- a/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py +++ b/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py @@ -68,7 +68,7 @@ source=PredictionSource("db", "t"), model="db.qwen_vl", inputs={"video": "content"}, - prompt="classify", + instructions="classify", output_schema={"result": "STRING"}, ), str: "SELECT", From b3642cb1cb2cbcb6788d4b490dfba50b413e826f Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Mon, 16 Nov 2026 21:51:21 +0800 Subject: [PATCH 12/41] [core][python] Standardize prediction result schema --- docs/static/rest-catalog-open-api.yaml | 12 ++++-------- .../org/apache/paimon/catalog/PredictionSpec.java | 7 ------- .../rest/requests/SubmitPredictionRequest.java | 9 --------- .../apache/paimon/rest/RESTApiPredictionTest.java | 11 +++++++---- .../paimon/rest/RESTCatalogPredictionTest.java | 4 ++-- paimon-python/pypaimon/api/prediction.py | 4 ---- paimon-python/pypaimon/catalog/prediction.py | 3 --- paimon-python/pypaimon/multimodal/table.py | 2 -- paimon-python/pypaimon/tests/rest/prediction_test.py | 5 +---- .../pypaimon/tests/rest/rest_api_coverage_test.py | 1 - 10 files changed, 14 insertions(+), 44 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 27e44311c5e8..9a8abc5835f7 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -4707,11 +4707,14 @@ components: properties: table: $ref: '#/components/schemas/PredictionSource' + description: >- + Managed result table containing source identity columns, a response string, + and an error string. additionalProperties: true SubmitPredictionRequest: type: object additionalProperties: false - required: [source, idempotencyKey, model, inputs, instructions, outputSchema] + required: [source, idempotencyKey, model, inputs, instructions] properties: source: $ref: '#/components/schemas/PredictionSource' @@ -4746,13 +4749,6 @@ components: type: string minLength: 1 description: Natural-language guidance for the prediction. - outputSchema: - type: object - minProperties: 1 - additionalProperties: - type: string - minLength: 1 - description: Paimon output column types. where: type: string minLength: 1 diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java index e10ef1330a6a..50f5895c9682 100644 --- a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java @@ -39,7 +39,6 @@ public class PredictionSpec { private final Map inputs; private final List mergeKeyColumns; private final String instructions; - private final Map outputSchema; private final @Nullable String where; private final @Nullable Long snapshotId; private final @Nullable String tagName; @@ -51,7 +50,6 @@ public PredictionSpec( Map inputs, @Nullable List mergeKeyColumns, String instructions, - Map outputSchema, @Nullable String where, @Nullable Long snapshotId, @Nullable String tagName, @@ -70,7 +68,6 @@ public PredictionSpec( this.inputs = copyStringMap(inputs, "inputs", true); this.mergeKeyColumns = copyOptionalStringList(mergeKeyColumns, "mergeKeyColumns"); this.instructions = instructions; - this.outputSchema = copyStringMap(outputSchema, "outputSchema", true); this.where = where; this.snapshotId = snapshotId; this.tagName = tagName; @@ -101,10 +98,6 @@ public String instructions() { return instructions; } - public Map outputSchema() { - return outputSchema; - } - @Nullable public String where() { return where; diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java index eaedb0e1b2aa..96d07249e365 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java @@ -48,7 +48,6 @@ public class SubmitPredictionRequest implements RESTRequest { private static final String FIELD_INPUTS = "inputs"; private static final String FIELD_MERGE_KEY_COLUMNS = "mergeKeyColumns"; private static final String FIELD_INSTRUCTIONS = "instructions"; - private static final String FIELD_OUTPUT_SCHEMA = "outputSchema"; private static final String FIELD_WHERE = "where"; private static final String FIELD_SNAPSHOT_ID = "snapshotId"; private static final String FIELD_TAG_NAME = "tagName"; @@ -67,7 +66,6 @@ public SubmitPredictionRequest( @JsonProperty(FIELD_INPUTS) Map inputs, @Nullable @JsonProperty(FIELD_MERGE_KEY_COLUMNS) List mergeKeyColumns, @JsonProperty(FIELD_INSTRUCTIONS) String instructions, - @JsonProperty(FIELD_OUTPUT_SCHEMA) Map outputSchema, @Nullable @JsonProperty(FIELD_WHERE) String where, @Nullable @JsonProperty(FIELD_SNAPSHOT_ID) Long snapshotId, @Nullable @JsonProperty(FIELD_TAG_NAME) String tagName, @@ -86,7 +84,6 @@ public SubmitPredictionRequest( inputs, mergeKeyColumns, instructions, - outputSchema, where, snapshotId, tagName, @@ -102,7 +99,6 @@ public SubmitPredictionRequest(Identifier source, PredictionSpec spec) { spec.inputs(), spec.mergeKeyColumns(), spec.instructions(), - spec.outputSchema(), spec.where(), spec.snapshotId(), spec.tagName(), @@ -147,11 +143,6 @@ public String getInstructions() { return spec.instructions(); } - @JsonGetter(FIELD_OUTPUT_SCHEMA) - public Map getOutputSchema() { - return spec.outputSchema(); - } - @Nullable @JsonInclude(JsonInclude.Include.NON_NULL) @JsonGetter(FIELD_WHERE) diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java index 278eea7b5d4a..ff9fdb471e28 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java @@ -54,7 +54,13 @@ void testRequestAndResponseWireFormat() throws Exception { .containsEntry("model", "quality.qwen_vl") .containsEntry("instructions", "Classify") .doesNotContainKeys( - "name", "prompt", "where", "snapshotId", "tagName", "resultColumns"); + "name", + "prompt", + "outputSchema", + "where", + "snapshotId", + "tagName", + "resultColumns"); assertThat(requestMap.get("mergeKeyColumns")) .isEqualTo(Collections.singletonList("clip_id")); assertThat((Map) requestMap.get("source")) @@ -123,7 +129,6 @@ void testSpecValidation() { Collections.emptyMap(), null, "Classify", - Collections.singletonMap("result", "STRING"), null, null, null, @@ -139,7 +144,6 @@ void testSpecValidation() { Collections.singletonMap("video", "content"), Arrays.asList("clip_id", "clip_id"), "Classify", - Collections.singletonMap("result", "STRING"), null, null, null, @@ -182,7 +186,6 @@ private static PredictionSpec spec() { inputs, Collections.singletonList("clip_id"), "Classify", - Collections.singletonMap("result", "STRING"), null, null, null, diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java index 58f078e9433c..64c032be1094 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java @@ -57,7 +57,8 @@ void testPredictionLifecycleUsesSharedRestContract() throws Exception { Map request = RESTApi.fromJson(submit.getBody().readUtf8(), Map.class); assertThat(request.get("idempotencyKey")).isInstanceOf(String.class); assertThat((String) request.get("idempotencyKey")).isNotBlank(); - assertThat(request).doesNotContainKeys("resultColumns", "mergeKeyColumns"); + assertThat(request) + .doesNotContainKeys("resultColumns", "mergeKeyColumns", "outputSchema"); assertThat((Map) request.get("source")) .containsEntry("database", "quality") .containsEntry("table", "clips"); @@ -118,7 +119,6 @@ private static PredictionSpec spec() { Collections.singletonMap("video", "content"), null, "Classify", - Collections.singletonMap("result", "STRING"), null, null, null, diff --git a/paimon-python/pypaimon/api/prediction.py b/paimon-python/pypaimon/api/prediction.py index 242b6f6f3a69..12b481096a97 100644 --- a/paimon-python/pypaimon/api/prediction.py +++ b/paimon-python/pypaimon/api/prediction.py @@ -83,7 +83,6 @@ class SubmitPredictionRequest(RESTRequest): model: str inputs: Dict[str, str] instructions: str - output_schema: Dict[str, str] = json_field("outputSchema") merge_key_columns: Optional[List[str]] = optional_json_field( "mergeKeyColumns", "non_null") name: Optional[str] = optional_json_field("name", "non_null") @@ -120,8 +119,6 @@ def __post_init__(self): if self.merge_key_columns is not None: self.merge_key_columns = _copy_string_list( self.merge_key_columns, "merge_key_columns") - self.output_schema = _copy_string_mapping( - self.output_schema, "output_schema", True) if not isinstance(self.options, dict): raise ValueError("options must be a mapping") self.options = dict(self.options) @@ -136,7 +133,6 @@ def from_spec( model=spec.model, inputs=spec.inputs, instructions=spec.instructions, - output_schema=spec.output_schema, merge_key_columns=spec.merge_key_columns or None, where=spec.where, snapshot_id=spec.snapshot_id, diff --git a/paimon-python/pypaimon/catalog/prediction.py b/paimon-python/pypaimon/catalog/prediction.py index 45536744a933..7d23106ca0ba 100644 --- a/paimon-python/pypaimon/catalog/prediction.py +++ b/paimon-python/pypaimon/catalog/prediction.py @@ -110,7 +110,6 @@ class PredictionSpec: model: str inputs: Dict[str, str] instructions: str - output_schema: Dict[str, str] merge_key_columns: List[str] = field(default_factory=list) name: Optional[str] = None where: Optional[str] = None @@ -136,8 +135,6 @@ def __post_init__(self): self.inputs = _copy_string_mapping(self.inputs, "inputs", True) self.merge_key_columns = _copy_string_list( self.merge_key_columns, "merge_key_columns") - self.output_schema = _copy_string_mapping( - self.output_schema, "output_schema", True) if not isinstance(self.options, dict): raise ValueError("options must be a mapping") self.options = dict(self.options) diff --git a/paimon-python/pypaimon/multimodal/table.py b/paimon-python/pypaimon/multimodal/table.py index f76afd349b62..2af8047ffacb 100644 --- a/paimon-python/pypaimon/multimodal/table.py +++ b/paimon-python/pypaimon/multimodal/table.py @@ -117,7 +117,6 @@ def predict( model, inputs, instructions, - output_schema, merge_key_columns=None, name=None, where=None, @@ -145,7 +144,6 @@ def predict( model=model, inputs=inputs, instructions=instructions, - output_schema=output_schema, merge_key_columns=( [] if merge_key_columns is None else merge_key_columns), where=where, diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index 978d223dfde3..5869b675cfaf 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -54,7 +54,6 @@ def _spec(**changes): model="quality.qwen_vl", inputs={"video": "content"}, instructions="Classify", - output_schema={"result": "STRING"}, ) values.update(changes) return PredictionSpec(**values) @@ -105,7 +104,7 @@ def test_request_and_response_wire_format(self): self.assertNotIn("mergeKeyColumns", body) self.assertNotIn("prompt", body) self.assertEqual("Classify", body["instructions"]) - self.assertEqual({"result": "STRING"}, body["outputSchema"]) + self.assertNotIn("outputSchema", body) self.assertEqual( {"database": "quality", "table": "clips"}, body["source"]) self.assertNotIn("snapshotId", body) @@ -190,7 +189,6 @@ def test_validation(self): dict(merge_key_columns=["id", "id"]), dict(merge_key_columns="id"), dict(instructions=" "), - dict(output_schema={}), dict(snapshot_id=0), dict(snapshot_id=1, tag_name="v1"), ] @@ -219,7 +217,6 @@ def test_table_submit_fixes_view_and_returns_job(self): model="quality.qwen_vl", inputs={"video": "content"}, instructions="Classify", - output_schema={"result": "STRING"}, ) self.assertIsInstance(job, PredictionJob) diff --git a/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py b/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py index 367d792e99f1..9f8699bcd323 100644 --- a/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py +++ b/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py @@ -69,7 +69,6 @@ model="db.qwen_vl", inputs={"video": "content"}, instructions="classify", - output_schema={"result": "STRING"}, ), str: "SELECT", bool: False, From 94f6cc38236e01d99b4e7cf81d7f0373fe1c13f2 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Mon, 16 Nov 2026 22:29:56 +0800 Subject: [PATCH 13/41] [python] Add prediction result waiter --- paimon-python/pypaimon/common/wait.py | 94 ++++++++++++++++ .../pypaimon/multimodal/prediction.py | 100 +++++++++--------- paimon-python/pypaimon/multimodal/table.py | 31 +++--- .../pypaimon/tests/rest/prediction_test.py | 87 ++++++++++++--- 4 files changed, 234 insertions(+), 78 deletions(-) create mode 100644 paimon-python/pypaimon/common/wait.py diff --git a/paimon-python/pypaimon/common/wait.py b/paimon-python/pypaimon/common/wait.py new file mode 100644 index 000000000000..f5971fa395e3 --- /dev/null +++ b/paimon-python/pypaimon/common/wait.py @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +from datetime import timedelta +from typing import Callable, Generic, Optional, TypeVar + + +T = TypeVar("T") + + +class Wait(Generic[T]): + """Client-side waiter for a long-running operation.""" + + def __init__( + self, + initial: T, + poll: Callable[[], T], + is_terminal: Callable[[T], bool], + validate_result: Callable[[T], T], + description: str = "Operation", + default_timeout: timedelta = timedelta(minutes=20), + initial_poll_interval: float = 1.0, + max_poll_interval: float = 10.0, + ): + if default_timeout.total_seconds() <= 0: + raise ValueError("default_timeout must be positive") + if initial_poll_interval <= 0: + raise ValueError("initial_poll_interval must be positive") + if max_poll_interval < initial_poll_interval: + raise ValueError( + "max_poll_interval must not be less than " + "initial_poll_interval") + self._current = initial + self._poll = poll + self._is_terminal = is_terminal + self._validate_result = validate_result + self._description = description + self._default_timeout = default_timeout + self._initial_poll_interval = initial_poll_interval + self._max_poll_interval = max_poll_interval + + @property + def current(self) -> T: + """Return the latest value observed by this waiter.""" + return self._current + + def result( + self, + timeout: Optional[timedelta] = None, + callback: Optional[Callable[[T], None]] = None, + ) -> T: + """Wait for completion and return the final value. + + The waiter polls through the supplied operation-specific callback. A + timeout only stops this client-side wait; it does not cancel the + server-side operation. + """ + if timeout is None: + timeout = self._default_timeout + if not isinstance(timeout, timedelta): + raise ValueError("timeout must be a datetime.timedelta") + if timeout.total_seconds() <= 0: + raise ValueError("timeout must be positive") + + deadline = time.monotonic() + timeout.total_seconds() + poll_interval = self._initial_poll_interval + while not self._is_terminal(self._current): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError( + "{} did not finish within {}".format( + self._description, timeout)) + time.sleep(min(poll_interval, remaining)) + self._current = self._poll() + if callback is not None: + callback(self._current) + poll_interval = min( + poll_interval * 2, self._max_poll_interval) + + return self._validate_result(self._current) diff --git a/paimon-python/pypaimon/multimodal/prediction.py b/paimon-python/pypaimon/multimodal/prediction.py index fe1364203c58..551948e08498 100644 --- a/paimon-python/pypaimon/multimodal/prediction.py +++ b/paimon-python/pypaimon/multimodal/prediction.py @@ -15,64 +15,66 @@ # specific language governing permissions and limitations # under the License. -import time -from datetime import timedelta -from typing import Optional - from pypaimon.catalog.prediction import Prediction +from pypaimon.catalog.prediction import PredictionTerminationCode +from pypaimon.common.wait import Wait -class PredictionJob: - """Client handle for a durable asynchronous prediction job.""" +class PredictionFailedError(RuntimeError): + """Raised when an asynchronous prediction fails.""" - def __init__(self, catalog, prediction: Prediction): - self._catalog = catalog - self._prediction = prediction + def __init__(self, prediction: Prediction): + details = prediction.status.termination_details + message = details.message if details is not None else None + super().__init__( + message or "Prediction {} failed".format( + prediction.prediction_id)) + self.prediction = prediction - @property - def prediction(self) -> Prediction: - return self._prediction - def __getattr__(self, name): - return getattr(self._prediction, name) +class PredictionCanceledError(RuntimeError): + """Raised when an asynchronous prediction is canceled.""" + + def __init__(self, prediction: Prediction): + details = prediction.status.termination_details + message = details.message if details is not None else None + super().__init__( + message or "Prediction {} was canceled".format( + prediction.prediction_id)) + self.prediction = prediction - def refresh(self) -> 'PredictionJob': - self._prediction = self._catalog.get_prediction( - self._prediction.prediction_id) - return self - def wait( - self, - timeout: Optional[timedelta] = None, - poll_interval: float = 1.0, - ) -> 'PredictionJob': - """Poll prediction metadata until it reaches a terminal state. +class PredictionWait(Wait[Prediction]): + """Wait handle returned after submitting an asynchronous prediction.""" - This is client-side polling over ``get_prediction``. It does not keep - a long-lived request open on the REST service. - """ - if timeout is not None and not isinstance(timeout, timedelta): - raise ValueError("timeout must be a datetime.timedelta") - if poll_interval < 0: - raise ValueError("poll_interval must not be negative") - deadline = ( - None if timeout is None - else time.monotonic() + timeout.total_seconds() + def __init__(self, catalog, prediction: Prediction): + self._catalog = catalog + prediction_id = prediction.prediction_id + super().__init__( + prediction, + lambda: catalog.get_prediction(prediction_id), + lambda value: value.status.terminal, + _validate_prediction_result, + "Prediction {}".format(prediction_id), ) - while not self._prediction.status.terminal: - if deadline is not None: - remaining = deadline - time.monotonic() - if remaining <= 0: - raise TimeoutError( - "Prediction {} did not finish within {}".format( - self._prediction.prediction_id, timeout)) - time.sleep(min(poll_interval, remaining)) - else: - time.sleep(poll_interval) - self.refresh() - return self - def cancel(self) -> 'PredictionJob': - self._prediction = self._catalog.cancel_prediction( - self._prediction.prediction_id) + @property + def prediction_id(self) -> str: + return self.current.prediction_id + + def cancel(self) -> 'PredictionWait': + self._current = self._catalog.cancel_prediction(self.prediction_id) return self + + +def _validate_prediction_result(prediction: Prediction) -> Prediction: + details = prediction.status.termination_details + if details is None: + raise RuntimeError( + "Terminal prediction {} has no termination details".format( + prediction.prediction_id)) + if details.code is PredictionTerminationCode.FAILED: + raise PredictionFailedError(prediction) + if details.code is PredictionTerminationCode.CANCELED: + raise PredictionCanceledError(prediction) + return prediction diff --git a/paimon-python/pypaimon/multimodal/table.py b/paimon-python/pypaimon/multimodal/table.py index 2af8047ffacb..563fec493ad6 100644 --- a/paimon-python/pypaimon/multimodal/table.py +++ b/paimon-python/pypaimon/multimodal/table.py @@ -123,7 +123,7 @@ def predict( options=None): """Submit an asynchronous prediction without reading rows locally.""" from pypaimon.catalog.prediction import PredictionSpec - from pypaimon.multimodal.prediction import PredictionJob + from pypaimon.multimodal.prediction import PredictionWait from pypaimon.table.file_store_table import FileStoreTable if not isinstance(self.raw_table, FileStoreTable): @@ -138,7 +138,8 @@ def predict( ] if unsupported: raise ValueError( - "Prediction does not support scan options: {}".format(unsupported)) + "Prediction does not support scan options: {}".format( + unsupported)) spec = PredictionSpec( name=name, model=model, @@ -153,14 +154,18 @@ def predict( ) prediction = self.catalog.submit_prediction( self.raw_table.identifier, spec) - return PredictionJob(self.catalog, prediction) + return PredictionWait(self.catalog, prediction) def get_prediction(self, prediction_id): - """Return a client handle for one asynchronous prediction.""" - from pypaimon.multimodal.prediction import PredictionJob + """Return metadata for one asynchronous prediction.""" + return self.catalog.get_prediction(prediction_id) - prediction = self.catalog.get_prediction(prediction_id) - return PredictionJob(self.catalog, prediction) + def cancel_prediction(self, prediction_id): + """Request cancellation and return a wait handle.""" + from pypaimon.multimodal.prediction import PredictionWait + + prediction = self.catalog.cancel_prediction(prediction_id) + return PredictionWait(self.catalog, prediction) def list_predictions( self, @@ -169,10 +174,8 @@ def list_predictions( state=None, max_results=None, page_token=None): - """List prediction handles for this table.""" - from pypaimon.api.api_response import PagedList + """List prediction metadata for this table.""" from pypaimon.catalog.prediction import PredictionState - from pypaimon.multimodal.prediction import PredictionJob if isinstance(state, str): state = PredictionState(state) @@ -183,13 +186,7 @@ def list_predictions( max_results=max_results, page_token=page_token, ) - return PagedList( - [ - PredictionJob(self.catalog, value) - for value in page.elements - ], - page.next_page_token, - ) + return page def add(self, data): arrow_table = _to_arrow_table(data, _target_schema(self.raw_table)) diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index 5869b675cfaf..bef887bd87c0 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -20,8 +20,9 @@ import unittest from datetime import timedelta from http.server import BaseHTTPRequestHandler, HTTPServer -from unittest.mock import Mock +from unittest.mock import Mock, patch +from pypaimon.api.api_response import PagedList from pypaimon.api.prediction import ( GetPredictionResponse, ListPredictionsResponse, @@ -44,7 +45,11 @@ from pypaimon.common.json_util import JSON from pypaimon.common.options.core_options import CoreOptions from pypaimon.common.options.options import Options -from pypaimon.multimodal.prediction import PredictionJob +from pypaimon.multimodal.prediction import ( + PredictionCanceledError, + PredictionFailedError, + PredictionWait, +) from pypaimon.multimodal.table import MultimodalTable from pypaimon.table.file_store_table import FileStoreTable @@ -204,7 +209,7 @@ def test_validation(self): PredictionTerminationCode.FAILED), ) - def test_table_submit_fixes_view_and_returns_job(self): + def test_table_submit_fixes_view_and_returns_waiter(self): raw = Mock(spec=FileStoreTable) raw.options = CoreOptions(Options({"scan.snapshot-id": "7"})) raw.identifier = Identifier.create("quality", "clips") @@ -212,22 +217,43 @@ def test_table_submit_fixes_view_and_returns_job(self): catalog.submit_prediction.return_value = _prediction() table = MultimodalTable(catalog, "quality.clips", raw) - job = table.predict( + waiter = table.predict( name="daily-quality", model="quality.qwen_vl", inputs={"video": "content"}, instructions="Classify", ) - self.assertIsInstance(job, PredictionJob) - self.assertEqual("prediction-1", job.prediction_id) + self.assertIsInstance(waiter, PredictionWait) + self.assertEqual("prediction-1", waiter.prediction_id) sent = catalog.submit_prediction.call_args[0][1] self.assertEqual(7, sent.snapshot_id) self.assertIsNone(sent.tag_name) self.assertEqual([], sent.merge_key_columns) raw.new_read_builder.assert_not_called() - def test_wait_is_client_side_polling(self): + def test_table_queries_return_metadata_and_cancel_returns_waiter(self): + raw = Mock(spec=FileStoreTable) + raw.identifier = Identifier.create("quality", "clips") + catalog = Mock() + prediction = _prediction() + page = PagedList([prediction], "next") + catalog.get_prediction.return_value = prediction + catalog.list_predictions.return_value = page + catalog.cancel_prediction.return_value = prediction + table = MultimodalTable(catalog, "quality.clips", raw) + + self.assertIs(prediction, table.get_prediction("prediction-1")) + self.assertIs( + page, + table.list_predictions(state=PredictionState.PENDING), + ) + waiter = table.cancel_prediction("prediction-1") + self.assertIsInstance(waiter, PredictionWait) + self.assertEqual("prediction-1", waiter.prediction_id) + + @patch("pypaimon.common.wait.time.sleep") + def test_result_is_client_side_polling(self, sleep): catalog = Mock() catalog.get_prediction.side_effect = [ _prediction(PredictionStatus(PredictionState.RUNNING)), @@ -237,15 +263,52 @@ def test_wait_is_client_side_polling(self): PredictionTerminationCode.SUCCESS), )), ] - job = PredictionJob(catalog, _prediction()) - result = job.wait(timeout=timedelta(seconds=1), poll_interval=0) - self.assertIs(job, result) - self.assertEqual(PredictionState.TERMINATED, job.status.state) + waiter = PredictionWait(catalog, _prediction()) + observed = [] + result = waiter.result( + timeout=timedelta(seconds=1), + callback=lambda prediction: observed.append( + prediction.status.state), + ) + self.assertIs(waiter.current, result) + self.assertEqual(PredictionState.TERMINATED, result.status.state) self.assertEqual( PredictionTerminationCode.SUCCESS, - job.status.termination_details.code, + result.status.termination_details.code, ) + self.assertEqual( + [PredictionState.RUNNING, PredictionState.TERMINATED], observed) self.assertEqual(2, catalog.get_prediction.call_count) + self.assertEqual(2, sleep.call_count) + + @patch("pypaimon.common.wait.time.sleep") + def test_result_raises_for_failed_and_canceled_predictions(self, sleep): + for code, error in ( + (PredictionTerminationCode.FAILED, PredictionFailedError), + (PredictionTerminationCode.CANCELED, PredictionCanceledError), + ): + with self.subTest(code=code): + catalog = Mock() + catalog.get_prediction.return_value = _prediction( + PredictionStatus( + PredictionState.TERMINATED, + PredictionTerminationDetails(code, "terminal"), + )) + waiter = PredictionWait(catalog, _prediction()) + with self.assertRaises(error): + waiter.result(timeout=timedelta(seconds=1)) + + @patch("pypaimon.common.wait.time.monotonic", side_effect=[0, 2]) + def test_result_timeout_does_not_cancel_prediction(self, monotonic): + catalog = Mock() + waiter = PredictionWait(catalog, _prediction()) + + with self.assertRaisesRegex( + TimeoutError, "Prediction prediction-1 did not finish"): + waiter.result(timeout=timedelta(seconds=1)) + + catalog.get_prediction.assert_not_called() + catalog.cancel_prediction.assert_not_called() def test_rest_round_trip_paths_and_filters(self): calls = [] From 4a46915119b3cb4228cc75bf69807152d0b46aad Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Mon, 16 Nov 2026 22:35:35 +0800 Subject: [PATCH 14/41] [python] Document prediction waiter usage --- paimon-python/pypaimon/common/wait.py | 3 ++- paimon-python/pypaimon/multimodal/table.py | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/paimon-python/pypaimon/common/wait.py b/paimon-python/pypaimon/common/wait.py index f5971fa395e3..15d02cd8ae1e 100644 --- a/paimon-python/pypaimon/common/wait.py +++ b/paimon-python/pypaimon/common/wait.py @@ -67,7 +67,8 @@ def result( The waiter polls through the supplied operation-specific callback. A timeout only stops this client-side wait; it does not cancel the - server-side operation. + server-side operation. Durable workflows should observe long-running + operations asynchronously instead of keeping a client blocked here. """ if timeout is None: timeout = self._default_timeout diff --git a/paimon-python/pypaimon/multimodal/table.py b/paimon-python/pypaimon/multimodal/table.py index 563fec493ad6..6feef5ee3696 100644 --- a/paimon-python/pypaimon/multimodal/table.py +++ b/paimon-python/pypaimon/multimodal/table.py @@ -121,7 +121,13 @@ def predict( name=None, where=None, options=None): - """Submit an asynchronous prediction without reading rows locally.""" + """Submit an asynchronous prediction without reading rows locally. + + The returned waiter exposes the prediction ID immediately. Calling + ``result()`` is optional and intended for interactive clients; durable + workflows should persist the ID and observe the prediction + asynchronously. + """ from pypaimon.catalog.prediction import PredictionSpec from pypaimon.multimodal.prediction import PredictionWait from pypaimon.table.file_store_table import FileStoreTable From 1d5e2bf0d01b824eb27d0563efef5815820371df Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Tue, 17 Nov 2026 18:01:43 +0800 Subject: [PATCH 15/41] [python] Align prediction waiter with long-running operations --- paimon-python/pypaimon/common/wait.py | 82 ++++--------------- .../pypaimon/multimodal/prediction.py | 80 ------------------ .../pypaimon/multimodal/prediction_wait.py | 67 +++++++++++++++ paimon-python/pypaimon/multimodal/table.py | 4 +- .../pypaimon/tests/rest/prediction_test.py | 38 ++++----- 5 files changed, 105 insertions(+), 166 deletions(-) delete mode 100644 paimon-python/pypaimon/multimodal/prediction.py create mode 100644 paimon-python/pypaimon/multimodal/prediction_wait.py diff --git a/paimon-python/pypaimon/common/wait.py b/paimon-python/pypaimon/common/wait.py index 15d02cd8ae1e..a2838ee1cf2b 100644 --- a/paimon-python/pypaimon/common/wait.py +++ b/paimon-python/pypaimon/common/wait.py @@ -14,82 +14,36 @@ # See the License for the specific language governing permissions and # limitations under the License. -import time from datetime import timedelta -from typing import Callable, Generic, Optional, TypeVar +from typing import Callable, Generic, TypeVar T = TypeVar("T") class Wait(Generic[T]): - """Client-side waiter for a long-running operation.""" + """Handle for a long-running operation.""" - def __init__( - self, - initial: T, - poll: Callable[[], T], - is_terminal: Callable[[T], bool], - validate_result: Callable[[T], T], - description: str = "Operation", - default_timeout: timedelta = timedelta(minutes=20), - initial_poll_interval: float = 1.0, - max_poll_interval: float = 10.0, - ): - if default_timeout.total_seconds() <= 0: - raise ValueError("default_timeout must be positive") - if initial_poll_interval <= 0: - raise ValueError("initial_poll_interval must be positive") - if max_poll_interval < initial_poll_interval: - raise ValueError( - "max_poll_interval must not be less than " - "initial_poll_interval") - self._current = initial - self._poll = poll - self._is_terminal = is_terminal - self._validate_result = validate_result - self._description = description - self._default_timeout = default_timeout - self._initial_poll_interval = initial_poll_interval - self._max_poll_interval = max_poll_interval + def __init__(self, waiter: Callable, response=None, **kwargs): + self.response = response + self._waiter = waiter + self._bind = kwargs + + def __getattr__(self, name): + return self._bind[name] - @property - def current(self) -> T: - """Return the latest value observed by this waiter.""" - return self._current + def bind(self): + return dict(self._bind) def result( self, - timeout: Optional[timedelta] = None, - callback: Optional[Callable[[T], None]] = None, + timeout: timedelta = timedelta(minutes=20), + callback: Callable[[T], None] = None, ) -> T: - """Wait for completion and return the final value. + """Delegate waiting and return the final value. - The waiter polls through the supplied operation-specific callback. A - timeout only stops this client-side wait; it does not cancel the - server-side operation. Durable workflows should observe long-running - operations asynchronously instead of keeping a client blocked here. + A timeout only stops this client-side wait; it does not cancel the + server-side operation. """ - if timeout is None: - timeout = self._default_timeout - if not isinstance(timeout, timedelta): - raise ValueError("timeout must be a datetime.timedelta") - if timeout.total_seconds() <= 0: - raise ValueError("timeout must be positive") - - deadline = time.monotonic() + timeout.total_seconds() - poll_interval = self._initial_poll_interval - while not self._is_terminal(self._current): - remaining = deadline - time.monotonic() - if remaining <= 0: - raise TimeoutError( - "{} did not finish within {}".format( - self._description, timeout)) - time.sleep(min(poll_interval, remaining)) - self._current = self._poll() - if callback is not None: - callback(self._current) - poll_interval = min( - poll_interval * 2, self._max_poll_interval) - - return self._validate_result(self._current) + kwargs = self._bind.copy() + return self._waiter(callback=callback, timeout=timeout, **kwargs) diff --git a/paimon-python/pypaimon/multimodal/prediction.py b/paimon-python/pypaimon/multimodal/prediction.py deleted file mode 100644 index 551948e08498..000000000000 --- a/paimon-python/pypaimon/multimodal/prediction.py +++ /dev/null @@ -1,80 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from pypaimon.catalog.prediction import Prediction -from pypaimon.catalog.prediction import PredictionTerminationCode -from pypaimon.common.wait import Wait - - -class PredictionFailedError(RuntimeError): - """Raised when an asynchronous prediction fails.""" - - def __init__(self, prediction: Prediction): - details = prediction.status.termination_details - message = details.message if details is not None else None - super().__init__( - message or "Prediction {} failed".format( - prediction.prediction_id)) - self.prediction = prediction - - -class PredictionCanceledError(RuntimeError): - """Raised when an asynchronous prediction is canceled.""" - - def __init__(self, prediction: Prediction): - details = prediction.status.termination_details - message = details.message if details is not None else None - super().__init__( - message or "Prediction {} was canceled".format( - prediction.prediction_id)) - self.prediction = prediction - - -class PredictionWait(Wait[Prediction]): - """Wait handle returned after submitting an asynchronous prediction.""" - - def __init__(self, catalog, prediction: Prediction): - self._catalog = catalog - prediction_id = prediction.prediction_id - super().__init__( - prediction, - lambda: catalog.get_prediction(prediction_id), - lambda value: value.status.terminal, - _validate_prediction_result, - "Prediction {}".format(prediction_id), - ) - - @property - def prediction_id(self) -> str: - return self.current.prediction_id - - def cancel(self) -> 'PredictionWait': - self._current = self._catalog.cancel_prediction(self.prediction_id) - return self - - -def _validate_prediction_result(prediction: Prediction) -> Prediction: - details = prediction.status.termination_details - if details is None: - raise RuntimeError( - "Terminal prediction {} has no termination details".format( - prediction.prediction_id)) - if details.code is PredictionTerminationCode.FAILED: - raise PredictionFailedError(prediction) - if details.code is PredictionTerminationCode.CANCELED: - raise PredictionCanceledError(prediction) - return prediction diff --git a/paimon-python/pypaimon/multimodal/prediction_wait.py b/paimon-python/pypaimon/multimodal/prediction_wait.py new file mode 100644 index 000000000000..111aab6b97c0 --- /dev/null +++ b/paimon-python/pypaimon/multimodal/prediction_wait.py @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import random +import time +from datetime import timedelta +from typing import Callable, Optional + +from pypaimon.catalog.prediction import Prediction +from pypaimon.common.wait import Wait + + +class PredictionWait(Wait[Prediction]): + """Wait handle returned after submitting an asynchronous prediction.""" + + def __init__(self, catalog, prediction: Prediction): + super().__init__( + lambda prediction_id, timeout, callback: _wait_for_prediction( + catalog, prediction_id, timeout, callback), + response=prediction, + prediction_id=prediction.prediction_id, + ) + + +def _wait_for_prediction( + catalog, + prediction_id: str, + timeout: timedelta, + callback: Optional[Callable[[Prediction], None]], +) -> Prediction: + if not isinstance(timeout, timedelta): + raise ValueError("timeout must be a datetime.timedelta") + if timeout.total_seconds() <= 0: + raise ValueError("timeout must be positive") + + deadline = time.monotonic() + timeout.total_seconds() + attempt = 1 + while time.monotonic() < deadline: + prediction = catalog.get_prediction(prediction_id) + if prediction.status.terminal: + return prediction + if callback is not None: + callback(prediction) + remaining = deadline - time.monotonic() + if remaining <= 0: + break + sleep_seconds = min(attempt, 10) + random.random() + time.sleep(min(sleep_seconds, remaining)) + attempt += 1 + + raise TimeoutError( + "Prediction {} did not finish within {}".format( + prediction_id, timeout)) diff --git a/paimon-python/pypaimon/multimodal/table.py b/paimon-python/pypaimon/multimodal/table.py index 6feef5ee3696..bc048b32d634 100644 --- a/paimon-python/pypaimon/multimodal/table.py +++ b/paimon-python/pypaimon/multimodal/table.py @@ -129,7 +129,7 @@ def predict( asynchronously. """ from pypaimon.catalog.prediction import PredictionSpec - from pypaimon.multimodal.prediction import PredictionWait + from pypaimon.multimodal.prediction_wait import PredictionWait from pypaimon.table.file_store_table import FileStoreTable if not isinstance(self.raw_table, FileStoreTable): @@ -168,7 +168,7 @@ def get_prediction(self, prediction_id): def cancel_prediction(self, prediction_id): """Request cancellation and return a wait handle.""" - from pypaimon.multimodal.prediction import PredictionWait + from pypaimon.multimodal.prediction_wait import PredictionWait prediction = self.catalog.cancel_prediction(prediction_id) return PredictionWait(self.catalog, prediction) diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index bef887bd87c0..41298a1ee727 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -45,11 +45,7 @@ from pypaimon.common.json_util import JSON from pypaimon.common.options.core_options import CoreOptions from pypaimon.common.options.options import Options -from pypaimon.multimodal.prediction import ( - PredictionCanceledError, - PredictionFailedError, - PredictionWait, -) +from pypaimon.multimodal.prediction_wait import PredictionWait from pypaimon.multimodal.table import MultimodalTable from pypaimon.table.file_store_table import FileStoreTable @@ -252,8 +248,9 @@ def test_table_queries_return_metadata_and_cancel_returns_waiter(self): self.assertIsInstance(waiter, PredictionWait) self.assertEqual("prediction-1", waiter.prediction_id) - @patch("pypaimon.common.wait.time.sleep") - def test_result_is_client_side_polling(self, sleep): + @patch("pypaimon.multimodal.prediction_wait.random.random", return_value=0) + @patch("pypaimon.multimodal.prediction_wait.time.sleep") + def test_result_is_client_side_polling(self, sleep, random): catalog = Mock() catalog.get_prediction.side_effect = [ _prediction(PredictionStatus(PredictionState.RUNNING)), @@ -266,26 +263,23 @@ def test_result_is_client_side_polling(self, sleep): waiter = PredictionWait(catalog, _prediction()) observed = [] result = waiter.result( - timeout=timedelta(seconds=1), + timeout=timedelta(seconds=10), callback=lambda prediction: observed.append( prediction.status.state), ) - self.assertIs(waiter.current, result) self.assertEqual(PredictionState.TERMINATED, result.status.state) self.assertEqual( PredictionTerminationCode.SUCCESS, result.status.termination_details.code, ) - self.assertEqual( - [PredictionState.RUNNING, PredictionState.TERMINATED], observed) + self.assertEqual([PredictionState.RUNNING], observed) self.assertEqual(2, catalog.get_prediction.call_count) - self.assertEqual(2, sleep.call_count) + sleep.assert_called_once_with(1) - @patch("pypaimon.common.wait.time.sleep") - def test_result_raises_for_failed_and_canceled_predictions(self, sleep): - for code, error in ( - (PredictionTerminationCode.FAILED, PredictionFailedError), - (PredictionTerminationCode.CANCELED, PredictionCanceledError), + def test_result_returns_failed_and_canceled_predictions(self): + for code in ( + PredictionTerminationCode.FAILED, + PredictionTerminationCode.CANCELED, ): with self.subTest(code=code): catalog = Mock() @@ -295,10 +289,14 @@ def test_result_raises_for_failed_and_canceled_predictions(self, sleep): PredictionTerminationDetails(code, "terminal"), )) waiter = PredictionWait(catalog, _prediction()) - with self.assertRaises(error): - waiter.result(timeout=timedelta(seconds=1)) + result = waiter.result(timeout=timedelta(seconds=1)) + self.assertEqual( + code, result.status.termination_details.code) - @patch("pypaimon.common.wait.time.monotonic", side_effect=[0, 2]) + @patch( + "pypaimon.multimodal.prediction_wait.time.monotonic", + side_effect=[0, 2], + ) def test_result_timeout_does_not_cancel_prediction(self, monotonic): catalog = Mock() waiter = PredictionWait(catalog, _prediction()) From 25881914a9604178f62fedc8a0e87951c855774b Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Tue, 17 Nov 2026 18:18:03 +0800 Subject: [PATCH 16/41] [rest] Keep prediction API REST-only --- .../paimon/rest/auth/DLFOpenApiActions.java | 4 ---- .../apache/paimon/rest/RESTApiCoverageTest.java | 9 ++++++++- .../paimon/rest/auth/DLFOpenApiActionsTest.java | 15 +++++++++++++-- .../pypaimon/api/auth/dlf_openapi_actions.py | 4 ---- .../pypaimon/tests/rest/rest_api_coverage_test.py | 9 +++++++-- 5 files changed, 28 insertions(+), 13 deletions(-) diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java index 727443d8d61e..109dcbd5b263 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/auth/DLFOpenApiActions.java @@ -51,10 +51,6 @@ final class DLFOpenApiActions { {"GET", "/v1/{prefix}/tables/id/{tableId}", "GetTableById"}, {"POST", "/v1/{prefix}/tables/rename", "RenameTable"}, {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/auth", "AuthTableQuery"}, - {"POST", "/v1/{prefix}/predictions", "SubmitPrediction"}, - {"GET", "/v1/{prefix}/predictions", "ListPredictions"}, - {"GET", "/v1/{prefix}/predictions/{predictionId}", "GetPrediction"}, - {"POST", "/v1/{prefix}/predictions/{predictionId}/cancel", "CancelPrediction"}, {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/commit", "CommitTable"}, { "POST", diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiCoverageTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiCoverageTest.java index 32da66f2abc2..e7c4d9cc9813 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiCoverageTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiCoverageTest.java @@ -79,7 +79,14 @@ class RESTApiCoverageTest { /** RESTApi methods with no entry in DLFOpenApiActions; they are signed without an action. */ - private static final Set WITHOUT_ACTION = Collections.emptySet(); + private static final Set WITHOUT_ACTION = + Collections.unmodifiableSet( + new TreeSet<>( + Arrays.asList( + "cancelPrediction", + "getPrediction", + "listPredictions", + "submitPrediction"))); /** A value per parameter type; "SELECT" is valid both as a name and as an access. */ private static final Map, Object> ARGUMENTS = arguments(); diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiActionsTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiActionsTest.java index 6d741fd6828a..d456239417fe 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiActionsTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiActionsTest.java @@ -26,6 +26,7 @@ import org.junit.jupiter.api.Test; import java.io.IOException; +import java.util.Arrays; import java.util.HashSet; import java.util.Set; import java.util.TreeSet; @@ -38,6 +39,15 @@ /** Test for {@link DLFOpenApiActions}. */ public class DLFOpenApiActionsTest { + private static final Set REST_ONLY_OPERATIONS = + new HashSet<>( + Arrays.asList( + OpenApiSpec.normalize("POST /v1/{prefix}/predictions"), + OpenApiSpec.normalize("GET /v1/{prefix}/predictions"), + OpenApiSpec.normalize("GET /v1/{prefix}/predictions/{predictionId}"), + OpenApiSpec.normalize( + "POST /v1/{prefix}/predictions/{predictionId}/cancel"))); + private final ResourcePaths paths = new ResourcePaths("clg-paimon-1"); /** Each template, filled in, must come back to its own action rather than an earlier one. */ @@ -49,7 +59,7 @@ public void testEveryOperationResolvesToItself() { assertEquals(operation[2], DLFOpenApiActions.resolve(operation[0], path), path); actions.add(operation[2]); } - assertEquals(83, actions.size()); + assertEquals(79, actions.size()); } /** The table and docs/static/rest-*-open-api.yaml must list the same operations. */ @@ -59,12 +69,13 @@ public void testOperationsMatchTheOpenApiSpec() throws IOException { for (String operation : OpenApiSpec.operations()) { spec.add(OpenApiSpec.normalize(operation)); } + spec.removeAll(REST_ONLY_OPERATIONS); Set registered = new TreeSet<>(); for (String[] operation : DLFOpenApiActions.operations()) { registered.add(OpenApiSpec.normalize(operation[0] + " " + operation[1])); } assertThat(registered) - .as("DLFOpenApiActions must register exactly the spec operations") + .as("DLFOpenApiActions must register the POP-enabled spec operations") .containsExactlyInAnyOrderElementsOf(spec); } diff --git a/paimon-python/pypaimon/api/auth/dlf_openapi_actions.py b/paimon-python/pypaimon/api/auth/dlf_openapi_actions.py index 86d8437bbb04..9491c9b32d90 100644 --- a/paimon-python/pypaimon/api/auth/dlf_openapi_actions.py +++ b/paimon-python/pypaimon/api/auth/dlf_openapi_actions.py @@ -102,10 +102,6 @@ ("POST", "/v1/{prefix}/databases/{database}/models", "CreateModel"), ("GET", "/v1/{prefix}/databases/{database}/models/{model}", "GetModel"), ("DELETE", "/v1/{prefix}/databases/{database}/models/{model}", "DropModel"), - ("POST", "/v1/{prefix}/predictions", "SubmitPrediction"), - ("GET", "/v1/{prefix}/predictions", "ListPredictions"), - ("GET", "/v1/{prefix}/predictions/{predictionId}", "GetPrediction"), - ("POST", "/v1/{prefix}/predictions/{predictionId}/cancel", "CancelPrediction"), ("GET", "/v1/{prefix}/permissions", "ListPermissionAssignments"), ("POST", "/v1/{prefix}/permissions/grant", "GrantPermissionAssignment"), ("POST", "/v1/{prefix}/permissions/revoke", "RevokePermissionAssignment"), diff --git a/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py b/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py index 9f8699bcd323..c63a597a95b8 100644 --- a/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py +++ b/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py @@ -45,8 +45,13 @@ from pypaimon.snapshot.snapshot_commit import PartitionStatistics from pypaimon.table.instant import Instant -# RESTApi methods with no entry in dlf_openapi_actions; they are signed without an action. -WITHOUT_ACTION = frozenset() +# REST-only APIs with no DLF POP action; they are signed without x-acs-action. +WITHOUT_ACTION = frozenset({ + "cancel_prediction", + "get_prediction", + "list_predictions", + "submit_prediction", +}) _SPEC_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), *[os.pardir] * 4, "docs", "static") _SPEC_FILES = ("rest-catalog-open-api.yaml", "rest-management-open-api.yaml") From 0c6ecc290a197cf82cc360e419a6f7fbfa4b2989 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Tue, 17 Nov 2026 19:31:54 +0800 Subject: [PATCH 17/41] [rest] Reuse Paimon options for prediction time travel --- docs/static/rest-catalog-open-api.yaml | 14 ++---- .../apache/paimon/catalog/PredictionSpec.java | 46 ++++++++----------- .../requests/SubmitPredictionRequest.java | 34 ++------------ .../paimon/rest/RESTApiPredictionTest.java | 39 +++++++++++++--- .../rest/RESTCatalogPredictionTest.java | 2 - paimon-python/pypaimon/api/prediction.py | 25 +++++----- paimon-python/pypaimon/catalog/prediction.py | 29 ++++++------ paimon-python/pypaimon/multimodal/table.py | 22 +++++++-- .../pypaimon/tests/rest/prediction_test.py | 21 +++++++-- 9 files changed, 121 insertions(+), 111 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 9a8abc5835f7..b9104debf499 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -4752,17 +4752,13 @@ components: where: type: string minLength: 1 - snapshotId: - type: integer - format: int64 - minimum: 1 - tagName: - type: string - minLength: 1 options: type: object - additionalProperties: true - description: snapshotId and tagName are mutually exclusive. + additionalProperties: + type: string + description: >- + Paimon dynamic options. Use scan.snapshot-id or scan.tag-name for time + travel; the two options are mutually exclusive. GetPredictionResponse: type: object additionalProperties: false diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java index 50f5895c9682..182187168574 100644 --- a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java @@ -18,6 +18,7 @@ package org.apache.paimon.catalog; +import org.apache.paimon.CoreOptions; import org.apache.paimon.annotation.Experimental; import javax.annotation.Nullable; @@ -30,7 +31,12 @@ import static org.apache.paimon.utils.Preconditions.checkArgument; -/** Provider-neutral specification for an asynchronous prediction over a table snapshot. */ +/** + * Provider-neutral specification for an asynchronous prediction over a table snapshot. + * + *

{@code options} uses Paimon dynamic table option names. Time travel uses {@code + * scan.snapshot-id} or {@code scan.tag-name}. + */ @Experimental public class PredictionSpec { @@ -40,9 +46,7 @@ public class PredictionSpec { private final List mergeKeyColumns; private final String instructions; private final @Nullable String where; - private final @Nullable Long snapshotId; - private final @Nullable String tagName; - private final Map options; + private final Map options; public PredictionSpec( @Nullable String name, @@ -51,30 +55,26 @@ public PredictionSpec( @Nullable List mergeKeyColumns, String instructions, @Nullable String where, - @Nullable Long snapshotId, - @Nullable String tagName, - @Nullable Map options) { + @Nullable Map options) { checkOptionalNonBlank(name, "name"); checkNonBlank(model, "model"); checkNonBlank(instructions, "instructions"); checkOptionalNonBlank(where, "where"); - checkOptionalNonBlank(tagName, "tagName"); - checkArgument(snapshotId == null || snapshotId > 0, "snapshotId must be positive"); - checkArgument( - snapshotId == null || tagName == null, - "snapshotId and tagName are mutually exclusive"); this.name = name; this.model = model; this.inputs = copyStringMap(inputs, "inputs", true); this.mergeKeyColumns = copyOptionalStringList(mergeKeyColumns, "mergeKeyColumns"); this.instructions = instructions; this.where = where; - this.snapshotId = snapshotId; - this.tagName = tagName; this.options = - options == null - ? Collections.emptyMap() - : Collections.unmodifiableMap(new LinkedHashMap<>(options)); + options == null ? Collections.emptyMap() : copyStringMap(options, "options", false); + CoreOptions coreOptions = new CoreOptions(this.options); + Long snapshotId = coreOptions.scanSnapshotId(); + String tagName = coreOptions.scanTagName(); + checkArgument(snapshotId == null || snapshotId > 0, "scan.snapshot-id must be positive"); + checkArgument( + snapshotId == null || tagName == null, + "scan.snapshot-id and scan.tag-name are mutually exclusive"); } @Nullable @@ -103,17 +103,7 @@ public String where() { return where; } - @Nullable - public Long snapshotId() { - return snapshotId; - } - - @Nullable - public String tagName() { - return tagName; - } - - public Map options() { + public Map options() { return options; } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java index 96d07249e365..cb00cac056dc 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java @@ -49,8 +49,6 @@ public class SubmitPredictionRequest implements RESTRequest { private static final String FIELD_MERGE_KEY_COLUMNS = "mergeKeyColumns"; private static final String FIELD_INSTRUCTIONS = "instructions"; private static final String FIELD_WHERE = "where"; - private static final String FIELD_SNAPSHOT_ID = "snapshotId"; - private static final String FIELD_TAG_NAME = "tagName"; private static final String FIELD_OPTIONS = "options"; private final String idempotencyKey; @@ -67,9 +65,7 @@ public SubmitPredictionRequest( @Nullable @JsonProperty(FIELD_MERGE_KEY_COLUMNS) List mergeKeyColumns, @JsonProperty(FIELD_INSTRUCTIONS) String instructions, @Nullable @JsonProperty(FIELD_WHERE) String where, - @Nullable @JsonProperty(FIELD_SNAPSHOT_ID) Long snapshotId, - @Nullable @JsonProperty(FIELD_TAG_NAME) String tagName, - @Nullable @JsonProperty(FIELD_OPTIONS) Map options) { + @Nullable @JsonProperty(FIELD_OPTIONS) Map options) { checkArgument(source != null, "source must not be null"); checkArgument( idempotencyKey == null || !idempotencyKey.trim().isEmpty(), @@ -79,15 +75,7 @@ public SubmitPredictionRequest( idempotencyKey == null ? UUID.randomUUID().toString() : idempotencyKey; this.spec = new PredictionSpec( - name, - model, - inputs, - mergeKeyColumns, - instructions, - where, - snapshotId, - tagName, - options); + name, model, inputs, mergeKeyColumns, instructions, where, options); } public SubmitPredictionRequest(Identifier source, PredictionSpec spec) { @@ -100,8 +88,6 @@ public SubmitPredictionRequest(Identifier source, PredictionSpec spec) { spec.mergeKeyColumns(), spec.instructions(), spec.where(), - spec.snapshotId(), - spec.tagName(), spec.options()); } @@ -150,22 +136,8 @@ public String getWhere() { return spec.where(); } - @Nullable - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonGetter(FIELD_SNAPSHOT_ID) - public Long getSnapshotId() { - return spec.snapshotId(); - } - - @Nullable - @JsonInclude(JsonInclude.Include.NON_NULL) - @JsonGetter(FIELD_TAG_NAME) - public String getTagName() { - return spec.tagName(); - } - @JsonGetter(FIELD_OPTIONS) - public Map getOptions() { + public Map getOptions() { return spec.options(); } diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java index ff9fdb471e28..b0c3dd1dae04 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java @@ -66,6 +66,8 @@ void testRequestAndResponseWireFormat() throws Exception { assertThat((Map) requestMap.get("source")) .containsEntry("database", "quality") .containsEntry("table", "clips"); + assertThat((Map) requestMap.get("options")) + .containsEntry("scan.snapshot-id", "7"); GetPredictionResponse response = new GetPredictionResponse(prediction()); GetPredictionResponse decoded = @@ -130,8 +132,6 @@ void testSpecValidation() { null, "Classify", null, - null, - null, Collections.emptyMap())) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("inputs must not be empty"); @@ -145,11 +145,38 @@ void testSpecValidation() { Arrays.asList("clip_id", "clip_id"), "Classify", null, - null, - null, Collections.emptyMap())) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("mergeKeyColumns must not contain duplicates"); + + assertThatThrownBy( + () -> + new PredictionSpec( + null, + "quality.qwen_vl", + Collections.singletonMap("video", "content"), + null, + "Classify", + null, + Collections.singletonMap("scan.snapshot-id", "0"))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("scan.snapshot-id must be positive"); + + Map conflictingOptions = new LinkedHashMap<>(); + conflictingOptions.put("scan.snapshot-id", "7"); + conflictingOptions.put("scan.tag-name", "training"); + assertThatThrownBy( + () -> + new PredictionSpec( + null, + "quality.qwen_vl", + Collections.singletonMap("video", "content"), + null, + "Classify", + null, + conflictingOptions)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("mutually exclusive"); } @Test @@ -187,9 +214,7 @@ private static PredictionSpec spec() { Collections.singletonList("clip_id"), "Classify", null, - null, - null, - Collections.emptyMap()); + Collections.singletonMap("scan.snapshot-id", "7")); } private static Prediction prediction() { diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java index 64c032be1094..407544f266cb 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java @@ -120,8 +120,6 @@ private static PredictionSpec spec() { null, "Classify", null, - null, - null, Collections.emptyMap()); } diff --git a/paimon-python/pypaimon/api/prediction.py b/paimon-python/pypaimon/api/prediction.py index 12b481096a97..abf247c41be1 100644 --- a/paimon-python/pypaimon/api/prediction.py +++ b/paimon-python/pypaimon/api/prediction.py @@ -89,9 +89,7 @@ class SubmitPredictionRequest(RESTRequest): idempotency_key: Optional[str] = optional_json_field( "idempotencyKey", "non_null") where: Optional[str] = optional_json_field("where", "non_null") - snapshot_id: Optional[int] = optional_json_field("snapshotId", "non_null") - tag_name: Optional[str] = optional_json_field("tagName", "non_null") - options: Dict[str, Any] = field(default_factory=dict) + options: Dict[str, str] = field(default_factory=dict) def __post_init__(self): for value, field_name in ( @@ -102,7 +100,6 @@ def __post_init__(self): for value, field_name in ( (self.name, "name"), (self.where, "where"), - (self.tag_name, "tag_name"), ): if value is not None: _nonblank(value, field_name) @@ -110,18 +107,20 @@ def __post_init__(self): self.idempotency_key = uuid.uuid4().hex else: _nonblank(self.idempotency_key, "idempotency_key") - if self.snapshot_id is not None and ( - type(self.snapshot_id) is not int or self.snapshot_id <= 0): - raise ValueError("snapshot_id must be a positive integer") - if self.snapshot_id is not None and self.tag_name is not None: - raise ValueError("snapshot_id and tag_name are mutually exclusive") self.inputs = _copy_string_mapping(self.inputs, "inputs", True) if self.merge_key_columns is not None: self.merge_key_columns = _copy_string_list( self.merge_key_columns, "merge_key_columns") - if not isinstance(self.options, dict): - raise ValueError("options must be a mapping") - self.options = dict(self.options) + self.options = _copy_string_mapping(self.options, "options", False) + PredictionSpec( + model=self.model, + inputs=self.inputs, + instructions=self.instructions, + merge_key_columns=self.merge_key_columns or [], + name=self.name, + where=self.where, + options=self.options, + ) @classmethod def from_spec( @@ -135,8 +134,6 @@ def from_spec( instructions=spec.instructions, merge_key_columns=spec.merge_key_columns or None, where=spec.where, - snapshot_id=spec.snapshot_id, - tag_name=spec.tag_name, options=spec.options, ) diff --git a/paimon-python/pypaimon/catalog/prediction.py b/paimon-python/pypaimon/catalog/prediction.py index 7d23106ca0ba..12cc4130ee2f 100644 --- a/paimon-python/pypaimon/catalog/prediction.py +++ b/paimon-python/pypaimon/catalog/prediction.py @@ -17,10 +17,11 @@ from dataclasses import dataclass, field from enum import Enum -from typing import Any, Dict, List, Optional +from typing import Dict, List, Optional from pypaimon.common.identifier import Identifier from pypaimon.common.json_util import optional_json_field +from pypaimon.common.options.core_options import CoreOptions class PredictionState(str, Enum): @@ -113,9 +114,7 @@ class PredictionSpec: merge_key_columns: List[str] = field(default_factory=list) name: Optional[str] = None where: Optional[str] = None - snapshot_id: Optional[int] = None - tag_name: Optional[str] = None - options: Dict[str, Any] = field(default_factory=dict) + options: Dict[str, str] = field(default_factory=dict) def __post_init__(self): _nonblank(self.model, "model") @@ -123,21 +122,25 @@ def __post_init__(self): for value, name in ( (self.name, "name"), (self.where, "where"), - (self.tag_name, "tag_name"), ): if value is not None: _nonblank(value, name) - if self.snapshot_id is not None and ( - type(self.snapshot_id) is not int or self.snapshot_id <= 0): - raise ValueError("snapshot_id must be a positive integer") - if self.snapshot_id is not None and self.tag_name is not None: - raise ValueError("snapshot_id and tag_name are mutually exclusive") self.inputs = _copy_string_mapping(self.inputs, "inputs", True) self.merge_key_columns = _copy_string_list( self.merge_key_columns, "merge_key_columns") - if not isinstance(self.options, dict): - raise ValueError("options must be a mapping") - self.options = dict(self.options) + self.options = _copy_string_mapping(self.options, "options", False) + snapshot_id = self.options.get(CoreOptions.SCAN_SNAPSHOT_ID.key()) + tag_name = self.options.get(CoreOptions.SCAN_TAG_NAME.key()) + if snapshot_id is not None: + try: + if int(snapshot_id) <= 0: + raise ValueError + except ValueError: + raise ValueError( + "scan.snapshot-id must be a positive integer") + if snapshot_id is not None and tag_name is not None: + raise ValueError( + "scan.snapshot-id and scan.tag-name are mutually exclusive") @dataclass diff --git a/paimon-python/pypaimon/multimodal/table.py b/paimon-python/pypaimon/multimodal/table.py index bc048b32d634..923a69f54a5d 100644 --- a/paimon-python/pypaimon/multimodal/table.py +++ b/paimon-python/pypaimon/multimodal/table.py @@ -123,6 +123,9 @@ def predict( options=None): """Submit an asynchronous prediction without reading rows locally. + ``options`` uses Paimon dynamic table option names. Time travel uses + ``scan.snapshot-id`` or ``scan.tag-name``. + The returned waiter exposes the prediction ID immediately. Calling ``result()`` is optional and intended for interactive clients; durable workflows should persist the ID and observe the prediction @@ -137,15 +140,30 @@ def predict( "Prediction requires a Paimon file store table") table_options = self.raw_table.options.options.to_map() allowed = {"scan.snapshot-id", "scan.tag-name"} + prediction_options = dict(options or {}) unsupported = [ key for key in table_options if (key.startswith("scan.") or key.startswith("incremental-")) and key not in allowed ] + unsupported.extend([ + key for key in prediction_options + if (key.startswith("scan.") or key.startswith("incremental-")) + and key not in allowed + ]) if unsupported: raise ValueError( "Prediction does not support scan options: {}".format( unsupported)) + for key in allowed: + table_value = table_options.get(key) + if table_value is None: + continue + request_value = prediction_options.get(key) + if request_value is not None and request_value != table_value: + raise ValueError( + "Conflicting prediction option: {}".format(key)) + prediction_options[key] = table_value spec = PredictionSpec( name=name, model=model, @@ -154,9 +172,7 @@ def predict( merge_key_columns=( [] if merge_key_columns is None else merge_key_columns), where=where, - snapshot_id=self.raw_table.options.scan_snapshot_id(), - tag_name=self.raw_table.options.scan_tag_name(), - options=options or {}, + options=prediction_options, ) prediction = self.catalog.submit_prediction( self.raw_table.identifier, spec) diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index 41298a1ee727..d379a212c9ef 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -109,6 +109,16 @@ def test_request_and_response_wire_format(self): self.assertEqual( {"database": "quality", "table": "clips"}, body["source"]) self.assertNotIn("snapshotId", body) + self.assertNotIn("tagName", body) + + versioned = SubmitPredictionRequest.from_spec( + Identifier.create("quality", "clips"), + _spec(options={"scan.tag-name": "training"}), + ) + versioned_body = json.loads(JSON.to_json(versioned)) + self.assertEqual( + {"scan.tag-name": "training"}, versioned_body["options"]) + self.assertNotIn("tagName", versioned_body) keyed = SubmitPredictionRequest.from_spec( Identifier.create("quality", "clips"), @@ -190,8 +200,12 @@ def test_validation(self): dict(merge_key_columns=["id", "id"]), dict(merge_key_columns="id"), dict(instructions=" "), - dict(snapshot_id=0), - dict(snapshot_id=1, tag_name="v1"), + dict(options={"scan.snapshot-id": "0"}), + dict(options={ + "scan.snapshot-id": "1", + "scan.tag-name": "v1", + }), + dict(options={"parallelism": 2}), ] for changes in invalid: with self.subTest(changes=changes), self.assertRaises(ValueError): @@ -223,8 +237,7 @@ def test_table_submit_fixes_view_and_returns_waiter(self): self.assertIsInstance(waiter, PredictionWait) self.assertEqual("prediction-1", waiter.prediction_id) sent = catalog.submit_prediction.call_args[0][1] - self.assertEqual(7, sent.snapshot_id) - self.assertIsNone(sent.tag_name) + self.assertEqual({"scan.snapshot-id": "7"}, sent.options) self.assertEqual([], sent.merge_key_columns) raw.new_read_builder.assert_not_called() From 54c9d0e75a69b86fbdfcfbf41203785cefb52563 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Tue, 17 Nov 2026 20:36:38 +0800 Subject: [PATCH 18/41] [rest] Align prediction validation contract --- docs/static/rest-catalog-open-api.yaml | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index b9104debf499..d44389ab1ee8 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -196,7 +196,7 @@ paths: operationId: getPrediction parameters: - {name: prefix, in: path, required: true, schema: {type: string}} - - {name: predictionId, in: path, required: true, schema: {type: string, minLength: 1}} + - {name: predictionId, in: path, required: true, schema: {type: string, minLength: 1, maxLength: 64}} responses: "200": description: Prediction metadata @@ -217,7 +217,7 @@ paths: unchanged. parameters: - {name: prefix, in: path, required: true, schema: {type: string}} - - {name: predictionId, in: path, required: true, schema: {type: string, minLength: 1}} + - {name: predictionId, in: path, required: true, schema: {type: string, minLength: 1, maxLength: 64}} responses: "200": description: Latest prediction metadata @@ -4699,8 +4699,8 @@ components: additionalProperties: false required: [database, table] properties: - database: {type: string, minLength: 1} - table: {type: string, minLength: 1} + database: {type: string, minLength: 1, maxLength: 128} + table: {type: string, minLength: 1, maxLength: 128} PredictionOutput: type: object minProperties: 1 @@ -4721,26 +4721,31 @@ components: name: type: string minLength: 1 + maxLength: 128 description: Optional non-unique label. idempotencyKey: type: string minLength: 1 + maxLength: 128 description: SDK-generated key reused for retries of one logical submission. model: type: string minLength: 1 + maxLength: 256 description: Database-qualified model name. inputs: type: object minProperties: 1 + propertyNames: {type: string, minLength: 1, maxLength: 128} additionalProperties: type: string minLength: 1 + maxLength: 1024 description: Model input-to-column mappings. mergeKeyColumns: type: array uniqueItems: true - items: {type: string, minLength: 1} + items: {type: string, minLength: 1, maxLength: 128} description: >- Stable source columns used when table primary keys or row IDs are unavailable. Omit to let the server derive identity; without stable identity the prediction @@ -4748,14 +4753,19 @@ components: instructions: type: string minLength: 1 + maxLength: 1048576 description: Natural-language guidance for the prediction. where: type: string minLength: 1 + maxLength: 1048576 options: type: object + propertyNames: {type: string, minLength: 1, maxLength: 128} additionalProperties: type: string + minLength: 1 + maxLength: 1024 description: >- Paimon dynamic options. Use scan.snapshot-id or scan.tag-name for time travel; the two options are mutually exclusive. @@ -4764,14 +4774,14 @@ components: additionalProperties: false required: [predictionId, source, status, snapshotId, model] properties: - predictionId: {type: string, minLength: 1} + predictionId: {type: string, minLength: 1, maxLength: 64} source: $ref: '#/components/schemas/PredictionSource' - name: {type: string, minLength: 1} + name: {type: string, minLength: 1, maxLength: 128} status: $ref: '#/components/schemas/PredictionStatus' snapshotId: {type: integer, format: int64, minimum: 1} - model: {type: string, minLength: 1} + model: {type: string, minLength: 1, maxLength: 256} output: $ref: '#/components/schemas/PredictionOutput' startTime: {type: integer, format: int64, minimum: 0} From 36452bd90cb3ebdd5d79742ff9130d09051ccf91 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Tue, 17 Nov 2026 22:04:19 +0800 Subject: [PATCH 19/41] Make prediction idempotency key optional --- docs/static/rest-catalog-open-api.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index d44389ab1ee8..72653e7f4b95 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -4714,7 +4714,7 @@ components: SubmitPredictionRequest: type: object additionalProperties: false - required: [source, idempotencyKey, model, inputs, instructions] + required: [source, model, inputs, instructions] properties: source: $ref: '#/components/schemas/PredictionSource' @@ -4727,7 +4727,9 @@ components: type: string minLength: 1 maxLength: 128 - description: SDK-generated key reused for retries of one logical submission. + description: >- + Optional key reused for retries of one logical submission. SDKs generate + one by default; requests without a key are independent submissions. model: type: string minLength: 1 From fadd4220e16114935a95f0a9019b5e8973018465 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Tue, 17 Nov 2026 22:14:08 +0800 Subject: [PATCH 20/41] Use structured model references for predictions --- docs/static/rest-catalog-open-api.yaml | 15 ++-- .../org/apache/paimon/catalog/Prediction.java | 8 +-- .../paimon/catalog/PredictionModel.java | 68 +++++++++++++++++++ .../apache/paimon/catalog/PredictionSpec.java | 8 +-- .../requests/SubmitPredictionRequest.java | 18 +++-- .../rest/responses/GetPredictionResponse.java | 19 ++++-- .../paimon/rest/RESTApiPredictionTest.java | 38 ++++++++--- .../org/apache/paimon/rest/RESTCatalog.java | 2 +- .../rest/RESTCatalogPredictionTest.java | 8 ++- paimon-python/pypaimon/api/prediction.py | 42 +++++++++--- paimon-python/pypaimon/catalog/prediction.py | 16 +++-- .../pypaimon/catalog/rest/rest_catalog.py | 2 +- .../pypaimon/tests/rest/prediction_test.py | 17 ++++- 13 files changed, 208 insertions(+), 53 deletions(-) create mode 100644 paimon-api/src/main/java/org/apache/paimon/catalog/PredictionModel.java diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 72653e7f4b95..eec266047687 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -4701,6 +4701,13 @@ components: properties: database: {type: string, minLength: 1, maxLength: 128} table: {type: string, minLength: 1, maxLength: 128} + PredictionModel: + type: object + additionalProperties: false + required: [database, model] + properties: + database: {type: string, minLength: 1, maxLength: 128} + model: {type: string, minLength: 1, maxLength: 128} PredictionOutput: type: object minProperties: 1 @@ -4731,10 +4738,7 @@ components: Optional key reused for retries of one logical submission. SDKs generate one by default; requests without a key are independent submissions. model: - type: string - minLength: 1 - maxLength: 256 - description: Database-qualified model name. + $ref: '#/components/schemas/PredictionModel' inputs: type: object minProperties: 1 @@ -4783,7 +4787,8 @@ components: status: $ref: '#/components/schemas/PredictionStatus' snapshotId: {type: integer, format: int64, minimum: 1} - model: {type: string, minLength: 1, maxLength: 256} + model: + $ref: '#/components/schemas/PredictionModel' output: $ref: '#/components/schemas/PredictionOutput' startTime: {type: integer, format: int64, minimum: 0} diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java index 569f6f5922f1..53a42e5eae18 100644 --- a/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java @@ -33,7 +33,7 @@ public class Prediction { private final @Nullable String name; private final PredictionStatus status; private final long snapshotId; - private final String model; + private final Identifier model; private final @Nullable PredictionOutput output; private final @Nullable Long startTime; private final @Nullable Long endTime; @@ -44,7 +44,7 @@ public Prediction( @Nullable String name, PredictionStatus status, long snapshotId, - String model, + Identifier model, @Nullable PredictionOutput output, @Nullable Long startTime, @Nullable Long endTime) { @@ -52,7 +52,7 @@ public Prediction( checkArgument(source != null, "source must not be null"); checkArgument(status != null, "status must not be null"); checkArgument(snapshotId > 0, "snapshotId must be positive"); - checkNonBlank(model, "model"); + checkArgument(model != null, "model must not be null"); checkOptionalNonNegative(startTime, "startTime"); checkOptionalNonNegative(endTime, "endTime"); this.predictionId = predictionId; @@ -87,7 +87,7 @@ public long snapshotId() { return snapshotId; } - public String model() { + public Identifier model() { return model; } diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionModel.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionModel.java new file mode 100644 index 000000000000..23cb47e2fef1 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionModel.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.catalog; + +import org.apache.paimon.annotation.Experimental; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Registered model used by a prediction job. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class PredictionModel { + + private final String database; + private final String model; + + @JsonCreator + public PredictionModel( + @JsonProperty("database") String database, @JsonProperty("model") String model) { + checkNonBlank(database, "database"); + checkNonBlank(model, "model"); + this.database = database; + this.model = model; + } + + public PredictionModel(Identifier identifier) { + this(identifier.getDatabaseName(), identifier.getObjectName()); + } + + @JsonGetter("database") + public String database() { + return database; + } + + @JsonGetter("model") + public String model() { + return model; + } + + public Identifier toIdentifier() { + return Identifier.create(database, model); + } + + private static void checkNonBlank(String value, String name) { + checkArgument(value != null && !value.trim().isEmpty(), "%s must not be blank", name); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java index 182187168574..3893b11c1549 100644 --- a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java @@ -41,7 +41,7 @@ public class PredictionSpec { private final @Nullable String name; - private final String model; + private final Identifier model; private final Map inputs; private final List mergeKeyColumns; private final String instructions; @@ -50,14 +50,14 @@ public class PredictionSpec { public PredictionSpec( @Nullable String name, - String model, + Identifier model, Map inputs, @Nullable List mergeKeyColumns, String instructions, @Nullable String where, @Nullable Map options) { checkOptionalNonBlank(name, "name"); - checkNonBlank(model, "model"); + checkArgument(model != null, "model must not be null"); checkNonBlank(instructions, "instructions"); checkOptionalNonBlank(where, "where"); this.name = name; @@ -82,7 +82,7 @@ public String name() { return name; } - public String model() { + public Identifier model() { return model; } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java index cb00cac056dc..e0f8809d6c77 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java @@ -19,6 +19,7 @@ package org.apache.paimon.rest.requests; import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.PredictionModel; import org.apache.paimon.catalog.PredictionSource; import org.apache.paimon.catalog.PredictionSpec; import org.apache.paimon.rest.RESTRequest; @@ -60,13 +61,14 @@ public SubmitPredictionRequest( @JsonProperty(FIELD_SOURCE) PredictionSource source, @Nullable @JsonProperty(FIELD_NAME) String name, @Nullable @JsonProperty(FIELD_IDEMPOTENCY_KEY) String idempotencyKey, - @JsonProperty(FIELD_MODEL) String model, + @JsonProperty(FIELD_MODEL) PredictionModel model, @JsonProperty(FIELD_INPUTS) Map inputs, @Nullable @JsonProperty(FIELD_MERGE_KEY_COLUMNS) List mergeKeyColumns, @JsonProperty(FIELD_INSTRUCTIONS) String instructions, @Nullable @JsonProperty(FIELD_WHERE) String where, @Nullable @JsonProperty(FIELD_OPTIONS) Map options) { checkArgument(source != null, "source must not be null"); + checkArgument(model != null, "model must not be null"); checkArgument( idempotencyKey == null || !idempotencyKey.trim().isEmpty(), "idempotencyKey must not be blank"); @@ -75,7 +77,13 @@ public SubmitPredictionRequest( idempotencyKey == null ? UUID.randomUUID().toString() : idempotencyKey; this.spec = new PredictionSpec( - name, model, inputs, mergeKeyColumns, instructions, where, options); + name, + model.toIdentifier(), + inputs, + mergeKeyColumns, + instructions, + where, + options); } public SubmitPredictionRequest(Identifier source, PredictionSpec spec) { @@ -83,7 +91,7 @@ public SubmitPredictionRequest(Identifier source, PredictionSpec spec) { new PredictionSource(source), spec.name(), null, - spec.model(), + new PredictionModel(spec.model()), spec.inputs(), spec.mergeKeyColumns(), spec.instructions(), @@ -109,8 +117,8 @@ public String getIdempotencyKey() { } @JsonGetter(FIELD_MODEL) - public String getModel() { - return spec.model(); + public PredictionModel getModel() { + return new PredictionModel(spec.model()); } @JsonGetter(FIELD_INPUTS) diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java index f07b8c6947b8..b6ba724e6e33 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java @@ -19,6 +19,7 @@ package org.apache.paimon.rest.responses; import org.apache.paimon.catalog.Prediction; +import org.apache.paimon.catalog.PredictionModel; import org.apache.paimon.catalog.PredictionOutput; import org.apache.paimon.catalog.PredictionSource; import org.apache.paimon.catalog.PredictionStatus; @@ -41,7 +42,7 @@ public class GetPredictionResponse implements RESTResponse { private final @Nullable String name; private final PredictionStatus status; private final long snapshotId; - private final String model; + private final PredictionModel model; private final @Nullable PredictionOutput output; private final @Nullable Long startTime; private final @Nullable Long endTime; @@ -53,7 +54,7 @@ public GetPredictionResponse( @Nullable @JsonProperty("name") String name, @JsonProperty("status") PredictionStatus status, @JsonProperty("snapshotId") long snapshotId, - @JsonProperty("model") String model, + @JsonProperty("model") PredictionModel model, @Nullable @JsonProperty("output") PredictionOutput output, @Nullable @JsonProperty("startTime") Long startTime, @Nullable @JsonProperty("endTime") Long endTime) { @@ -75,7 +76,7 @@ public GetPredictionResponse(Prediction prediction) { prediction.name(), prediction.status(), prediction.snapshotId(), - prediction.model(), + new PredictionModel(prediction.model()), prediction.output(), prediction.startTime(), prediction.endTime()); @@ -109,7 +110,7 @@ public long getSnapshotId() { } @JsonGetter("model") - public String getModel() { + public PredictionModel getModel() { return model; } @@ -136,6 +137,14 @@ public Long getEndTime() { public Prediction toPrediction() { return new Prediction( - predictionId, source, name, status, snapshotId, model, output, startTime, endTime); + predictionId, + source, + name, + status, + snapshotId, + model.toIdentifier(), + output, + startTime, + endTime); } } diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java index b0c3dd1dae04..6e8437cdeaf4 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java @@ -51,7 +51,6 @@ void testRequestAndResponseWireFormat() throws Exception { assertThat(RESTApi.fromJson(RESTApi.toJson(request), Map.class).get("idempotencyKey")) .isEqualTo(requestMap.get("idempotencyKey")); assertThat(requestMap) - .containsEntry("model", "quality.qwen_vl") .containsEntry("instructions", "Classify") .doesNotContainKeys( "name", @@ -66,6 +65,9 @@ void testRequestAndResponseWireFormat() throws Exception { assertThat((Map) requestMap.get("source")) .containsEntry("database", "quality") .containsEntry("table", "clips"); + assertThat((Map) requestMap.get("model")) + .containsEntry("database", "quality") + .containsEntry("model", "qwen_vl"); assertThat((Map) requestMap.get("options")) .containsEntry("scan.snapshot-id", "7"); @@ -98,7 +100,7 @@ void testRequestAndResponseWireFormat() throws Exception { new PredictionStatus.TerminationDetails( PredictionStatus.TerminationCode.SUCCESS, null)), 7, - "quality.qwen_vl", + Identifier.create("quality", "qwen_vl"), new PredictionOutput( new PredictionSource("system", "prediction_results")), 10L, @@ -121,13 +123,33 @@ void testRequestAndResponseWireFormat() throws Exception { .containsEntry("predictions", Collections.emptyList()); } + @Test + void testStructuredModelPreservesDots() throws Exception { + PredictionSpec spec = + new PredictionSpec( + null, + Identifier.create("quality.archive", "qwen.vl"), + Collections.singletonMap("video", "content"), + null, + "Classify", + null, + Collections.emptyMap()); + SubmitPredictionRequest request = + new SubmitPredictionRequest(Identifier.create("quality", "clips"), spec); + SubmitPredictionRequest decoded = + RESTApi.fromJson(RESTApi.toJson(request), SubmitPredictionRequest.class); + + assertThat(decoded.getModel().toIdentifier()) + .isEqualTo(Identifier.create("quality.archive", "qwen.vl")); + } + @Test void testSpecValidation() { assertThatThrownBy( () -> new PredictionSpec( null, - "quality.qwen_vl", + Identifier.create("quality", "qwen_vl"), Collections.emptyMap(), null, "Classify", @@ -140,7 +162,7 @@ void testSpecValidation() { () -> new PredictionSpec( null, - "quality.qwen_vl", + Identifier.create("quality", "qwen_vl"), Collections.singletonMap("video", "content"), Arrays.asList("clip_id", "clip_id"), "Classify", @@ -153,7 +175,7 @@ void testSpecValidation() { () -> new PredictionSpec( null, - "quality.qwen_vl", + Identifier.create("quality", "qwen_vl"), Collections.singletonMap("video", "content"), null, "Classify", @@ -169,7 +191,7 @@ void testSpecValidation() { () -> new PredictionSpec( null, - "quality.qwen_vl", + Identifier.create("quality", "qwen_vl"), Collections.singletonMap("video", "content"), null, "Classify", @@ -209,7 +231,7 @@ private static PredictionSpec spec() { inputs.put("video", "content"); return new PredictionSpec( null, - "quality.qwen_vl", + Identifier.create("quality", "qwen_vl"), inputs, Collections.singletonList("clip_id"), "Classify", @@ -224,7 +246,7 @@ private static Prediction prediction() { null, new PredictionStatus(PredictionStatus.State.PENDING, null), 7, - "quality.qwen_vl", + Identifier.create("quality", "qwen_vl"), null, null, null); diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 0cdad8ed297f..72d3d25d04b1 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -1222,7 +1222,7 @@ public Prediction submitPrediction(Identifier identifier, PredictionSpec spec) .toPrediction(); } catch (NoSuchResourceException e) { if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_MODEL)) { - throw new ModelNotExistException(Identifier.fromString(spec.model()), e); + throw new ModelNotExistException(spec.model(), e); } throw new TableNotExistException(identifier, e); } catch (NotImplementedException e) { diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java index 407544f266cb..b5481c90741e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java @@ -115,7 +115,7 @@ void testPredictionNotFoundPreservesResourceType() throws Exception { private static PredictionSpec spec() { return new PredictionSpec( "daily-quality", - "quality.qwen_vl", + Identifier.create("quality", "qwen_vl"), Collections.singletonMap("video", "content"), null, "Classify", @@ -136,7 +136,8 @@ private static String predictionJson(String status) { return "{\"predictionId\":\"prediction-1\",\"source\":{\"database\":\"quality\"," + "\"table\":\"clips\"},\"name\":\"daily-quality\",\"status\":{\"state\":\"" + status - + "\"},\"snapshotId\":7,\"model\":\"quality.qwen_vl\"}"; + + "\"},\"snapshotId\":7,\"model\":{\"database\":\"quality\"," + + "\"model\":\"qwen_vl\"}}"; } private static String terminatedPredictionJson(String code) { @@ -144,7 +145,8 @@ private static String terminatedPredictionJson(String code) { + "\"table\":\"clips\"},\"name\":\"daily-quality\"," + "\"status\":{\"state\":\"TERMINATED\",\"terminationDetails\":{\"code\":\"" + code - + "\"}},\"snapshotId\":7,\"model\":\"quality.qwen_vl\"," + + "\"}},\"snapshotId\":7,\"model\":{\"database\":\"quality\"," + + "\"model\":\"qwen_vl\"}," + "\"output\":{\"table\":{" + "\"database\":\"system\",\"table\":\"results\"}}," + "\"startTime\":10,\"endTime\":20}"; diff --git a/paimon-python/pypaimon/api/prediction.py b/paimon-python/pypaimon/api/prediction.py index abf247c41be1..267284ac6fe2 100644 --- a/paimon-python/pypaimon/api/prediction.py +++ b/paimon-python/pypaimon/api/prediction.py @@ -52,6 +52,24 @@ def to_identifier(self) -> Identifier: return Identifier.create(self.database, self.table) +@dataclass +class PredictionModel: + database: str + model: str + + def __post_init__(self): + _nonblank(self.database, "database") + _nonblank(self.model, "model") + + @classmethod + def from_identifier(cls, identifier: Identifier) -> "PredictionModel": + return cls( + identifier.get_database_name(), identifier.get_object_name()) + + def to_identifier(self) -> Identifier: + return Identifier.create(self.database, self.model) + + @dataclass class PredictionOutput: table: Optional[PredictionSource] = optional_json_field( @@ -80,7 +98,7 @@ def to_catalog_output(self) -> CatalogPredictionOutput: @dataclass class SubmitPredictionRequest(RESTRequest): source: PredictionSource - model: str + model: PredictionModel inputs: Dict[str, str] instructions: str merge_key_columns: Optional[List[str]] = optional_json_field( @@ -92,11 +110,9 @@ class SubmitPredictionRequest(RESTRequest): options: Dict[str, str] = field(default_factory=dict) def __post_init__(self): - for value, field_name in ( - (self.model, "model"), - (self.instructions, "instructions"), - ): - _nonblank(value, field_name) + if not isinstance(self.model, PredictionModel): + raise ValueError("model must be a PredictionModel") + _nonblank(self.instructions, "instructions") for value, field_name in ( (self.name, "name"), (self.where, "where"), @@ -113,7 +129,7 @@ def __post_init__(self): self.merge_key_columns, "merge_key_columns") self.options = _copy_string_mapping(self.options, "options", False) PredictionSpec( - model=self.model, + model=self.model.to_identifier(), inputs=self.inputs, instructions=self.instructions, merge_key_columns=self.merge_key_columns or [], @@ -129,7 +145,7 @@ def from_spec( return cls( source=PredictionSource.from_identifier(source), name=spec.name, - model=spec.model, + model=PredictionModel.from_identifier(spec.model), inputs=spec.inputs, instructions=spec.instructions, merge_key_columns=spec.merge_key_columns or None, @@ -151,7 +167,7 @@ class GetPredictionResponse(RESTResponse): source: PredictionSource = None status: PredictionStatus = None snapshot_id: int = json_field("snapshotId", default=0) - model: str = "" + model: PredictionModel = None name: Optional[str] = optional_json_field("name", "non_null") output: Optional[PredictionOutput] = optional_json_field( "output", "non_null") @@ -170,7 +186,11 @@ def from_dict(cls, data: Dict[str, Any]) -> "GetPredictionResponse": name=data.get("name"), status=PredictionStatus.from_dict(data.get("status")), snapshot_id=data.get("snapshotId"), - model=data.get("model"), + model=( + PredictionModel(**data.get("model")) + if isinstance(data.get("model"), dict) + else data.get("model") + ), output=(PredictionOutput.from_dict(data.get("output")) if isinstance(data.get("output"), dict) else data.get("output")), @@ -185,7 +205,7 @@ def to_prediction(self) -> Prediction: name=self.name, status=self.status, snapshot_id=self.snapshot_id, - model=self.model, + model=self.model.to_identifier(), output=(self.output.to_catalog_output() if self.output is not None else None), start_time=self.start_time, diff --git a/paimon-python/pypaimon/catalog/prediction.py b/paimon-python/pypaimon/catalog/prediction.py index 12cc4130ee2f..366e49991224 100644 --- a/paimon-python/pypaimon/catalog/prediction.py +++ b/paimon-python/pypaimon/catalog/prediction.py @@ -17,7 +17,7 @@ from dataclasses import dataclass, field from enum import Enum -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Union from pypaimon.common.identifier import Identifier from pypaimon.common.json_util import optional_json_field @@ -108,7 +108,7 @@ def terminal(self) -> bool: class PredictionSpec: """Provider-neutral input for an asynchronous table prediction.""" - model: str + model: Union[str, Identifier] inputs: Dict[str, str] instructions: str merge_key_columns: List[str] = field(default_factory=list) @@ -117,7 +117,10 @@ class PredictionSpec: options: Dict[str, str] = field(default_factory=dict) def __post_init__(self): - _nonblank(self.model, "model") + if isinstance(self.model, str): + self.model = Identifier.from_string(self.model) + if not isinstance(self.model, Identifier): + raise ValueError("model must be an Identifier or qualified name") _nonblank(self.instructions, "instructions") for value, name in ( (self.name, "name"), @@ -162,7 +165,7 @@ class Prediction: source: Identifier status: PredictionStatus snapshot_id: int - model: str + model: Union[str, Identifier] name: Optional[str] = None output: Optional[PredictionOutput] = None start_time: Optional[int] = None @@ -176,7 +179,10 @@ def __post_init__(self): raise ValueError("status must be a PredictionStatus") if type(self.snapshot_id) is not int or self.snapshot_id <= 0: raise ValueError("snapshot_id must be a positive integer") - _nonblank(self.model, "model") + if isinstance(self.model, str): + self.model = Identifier.from_string(self.model) + if not isinstance(self.model, Identifier): + raise ValueError("model must be an Identifier or qualified name") if self.output is not None and not isinstance( self.output, PredictionOutput): raise ValueError("output must be a PredictionOutput") diff --git a/paimon-python/pypaimon/catalog/rest/rest_catalog.py b/paimon-python/pypaimon/catalog/rest/rest_catalog.py index afdbaaffaa3b..187cbd962181 100644 --- a/paimon-python/pypaimon/catalog/rest/rest_catalog.py +++ b/paimon-python/pypaimon/catalog/rest/rest_catalog.py @@ -695,7 +695,7 @@ def submit_prediction( SubmitPredictionRequest.from_spec(identifier, spec)).to_prediction() except NoSuchResourceException as e: if e.resource_type == ErrorResponse.RESOURCE_TYPE_MODEL: - raise ModelNotExistException(Identifier.from_string(spec.model)) from e + raise ModelNotExistException(spec.model) from e raise TableNotExistException(identifier) from e except NotImplementedException as e: raise NotImplementedError( diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index d379a212c9ef..31a5e4c41bcb 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -27,6 +27,7 @@ GetPredictionResponse, ListPredictionsResponse, PredictionOutput as ApiPredictionOutput, + PredictionModel, PredictionSource, SubmitPredictionRequest, ) @@ -84,7 +85,7 @@ def _response(status=None, **changes): name="daily-quality", status=status, snapshot_id=7, - model="quality.qwen_vl", + model=PredictionModel("quality", "qwen_vl"), ) values.update(changes) return GetPredictionResponse(**values) @@ -108,6 +109,8 @@ def test_request_and_response_wire_format(self): self.assertNotIn("outputSchema", body) self.assertEqual( {"database": "quality", "table": "clips"}, body["source"]) + self.assertEqual( + {"database": "quality", "model": "qwen_vl"}, body["model"]) self.assertNotIn("snapshotId", body) self.assertNotIn("tagName", body) @@ -219,6 +222,18 @@ def test_validation(self): PredictionTerminationCode.FAILED), ) + def test_structured_model_preserves_dots(self): + request = SubmitPredictionRequest.from_spec( + Identifier.create("quality", "clips"), + _spec(model=Identifier.create("quality.archive", "qwen.vl")), + ) + body = json.loads(JSON.to_json(request)) + + self.assertEqual( + {"database": "quality.archive", "model": "qwen.vl"}, + body["model"], + ) + def test_table_submit_fixes_view_and_returns_waiter(self): raw = Mock(spec=FileStoreTable) raw.options = CoreOptions(Options({"scan.snapshot-id": "7"})) From b64f3505e45ac480aa1eea3d5dca0399580b9d01 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Tue, 17 Nov 2026 22:25:45 +0800 Subject: [PATCH 21/41] Reuse identifiers for prediction models --- docs/static/rest-catalog-open-api.yaml | 11 +-- .../paimon/catalog/PredictionModel.java | 68 ------------------- .../requests/SubmitPredictionRequest.java | 17 ++--- .../rest/responses/GetPredictionResponse.java | 20 ++---- .../paimon/rest/RESTApiPredictionTest.java | 5 +- .../rest/RESTCatalogPredictionTest.java | 4 +- paimon-python/pypaimon/api/prediction.py | 34 +++------- .../pypaimon/tests/rest/prediction_test.py | 7 +- 8 files changed, 28 insertions(+), 138 deletions(-) delete mode 100644 paimon-api/src/main/java/org/apache/paimon/catalog/PredictionModel.java diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index eec266047687..b2df4fbf5d99 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -4701,13 +4701,6 @@ components: properties: database: {type: string, minLength: 1, maxLength: 128} table: {type: string, minLength: 1, maxLength: 128} - PredictionModel: - type: object - additionalProperties: false - required: [database, model] - properties: - database: {type: string, minLength: 1, maxLength: 128} - model: {type: string, minLength: 1, maxLength: 128} PredictionOutput: type: object minProperties: 1 @@ -4738,7 +4731,7 @@ components: Optional key reused for retries of one logical submission. SDKs generate one by default; requests without a key are independent submissions. model: - $ref: '#/components/schemas/PredictionModel' + $ref: '#/components/schemas/Identifier' inputs: type: object minProperties: 1 @@ -4788,7 +4781,7 @@ components: $ref: '#/components/schemas/PredictionStatus' snapshotId: {type: integer, format: int64, minimum: 1} model: - $ref: '#/components/schemas/PredictionModel' + $ref: '#/components/schemas/Identifier' output: $ref: '#/components/schemas/PredictionOutput' startTime: {type: integer, format: int64, minimum: 0} diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionModel.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionModel.java deleted file mode 100644 index 23cb47e2fef1..000000000000 --- a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionModel.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.paimon.catalog; - -import org.apache.paimon.annotation.Experimental; - -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; - -import static org.apache.paimon.utils.Preconditions.checkArgument; - -/** Registered model used by a prediction job. */ -@Experimental -@JsonIgnoreProperties(ignoreUnknown = true) -public class PredictionModel { - - private final String database; - private final String model; - - @JsonCreator - public PredictionModel( - @JsonProperty("database") String database, @JsonProperty("model") String model) { - checkNonBlank(database, "database"); - checkNonBlank(model, "model"); - this.database = database; - this.model = model; - } - - public PredictionModel(Identifier identifier) { - this(identifier.getDatabaseName(), identifier.getObjectName()); - } - - @JsonGetter("database") - public String database() { - return database; - } - - @JsonGetter("model") - public String model() { - return model; - } - - public Identifier toIdentifier() { - return Identifier.create(database, model); - } - - private static void checkNonBlank(String value, String name) { - checkArgument(value != null && !value.trim().isEmpty(), "%s must not be blank", name); - } -} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java index e0f8809d6c77..239efc7ffb2e 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java @@ -19,7 +19,6 @@ package org.apache.paimon.rest.requests; import org.apache.paimon.catalog.Identifier; -import org.apache.paimon.catalog.PredictionModel; import org.apache.paimon.catalog.PredictionSource; import org.apache.paimon.catalog.PredictionSpec; import org.apache.paimon.rest.RESTRequest; @@ -61,7 +60,7 @@ public SubmitPredictionRequest( @JsonProperty(FIELD_SOURCE) PredictionSource source, @Nullable @JsonProperty(FIELD_NAME) String name, @Nullable @JsonProperty(FIELD_IDEMPOTENCY_KEY) String idempotencyKey, - @JsonProperty(FIELD_MODEL) PredictionModel model, + @JsonProperty(FIELD_MODEL) Identifier model, @JsonProperty(FIELD_INPUTS) Map inputs, @Nullable @JsonProperty(FIELD_MERGE_KEY_COLUMNS) List mergeKeyColumns, @JsonProperty(FIELD_INSTRUCTIONS) String instructions, @@ -77,13 +76,7 @@ public SubmitPredictionRequest( idempotencyKey == null ? UUID.randomUUID().toString() : idempotencyKey; this.spec = new PredictionSpec( - name, - model.toIdentifier(), - inputs, - mergeKeyColumns, - instructions, - where, - options); + name, model, inputs, mergeKeyColumns, instructions, where, options); } public SubmitPredictionRequest(Identifier source, PredictionSpec spec) { @@ -91,7 +84,7 @@ public SubmitPredictionRequest(Identifier source, PredictionSpec spec) { new PredictionSource(source), spec.name(), null, - new PredictionModel(spec.model()), + spec.model(), spec.inputs(), spec.mergeKeyColumns(), spec.instructions(), @@ -117,8 +110,8 @@ public String getIdempotencyKey() { } @JsonGetter(FIELD_MODEL) - public PredictionModel getModel() { - return new PredictionModel(spec.model()); + public Identifier getModel() { + return spec.model(); } @JsonGetter(FIELD_INPUTS) diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java index b6ba724e6e33..a0661fc814ab 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java @@ -18,8 +18,8 @@ package org.apache.paimon.rest.responses; +import org.apache.paimon.catalog.Identifier; import org.apache.paimon.catalog.Prediction; -import org.apache.paimon.catalog.PredictionModel; import org.apache.paimon.catalog.PredictionOutput; import org.apache.paimon.catalog.PredictionSource; import org.apache.paimon.catalog.PredictionStatus; @@ -42,7 +42,7 @@ public class GetPredictionResponse implements RESTResponse { private final @Nullable String name; private final PredictionStatus status; private final long snapshotId; - private final PredictionModel model; + private final Identifier model; private final @Nullable PredictionOutput output; private final @Nullable Long startTime; private final @Nullable Long endTime; @@ -54,7 +54,7 @@ public GetPredictionResponse( @Nullable @JsonProperty("name") String name, @JsonProperty("status") PredictionStatus status, @JsonProperty("snapshotId") long snapshotId, - @JsonProperty("model") PredictionModel model, + @JsonProperty("model") Identifier model, @Nullable @JsonProperty("output") PredictionOutput output, @Nullable @JsonProperty("startTime") Long startTime, @Nullable @JsonProperty("endTime") Long endTime) { @@ -76,7 +76,7 @@ public GetPredictionResponse(Prediction prediction) { prediction.name(), prediction.status(), prediction.snapshotId(), - new PredictionModel(prediction.model()), + prediction.model(), prediction.output(), prediction.startTime(), prediction.endTime()); @@ -110,7 +110,7 @@ public long getSnapshotId() { } @JsonGetter("model") - public PredictionModel getModel() { + public Identifier getModel() { return model; } @@ -137,14 +137,6 @@ public Long getEndTime() { public Prediction toPrediction() { return new Prediction( - predictionId, - source, - name, - status, - snapshotId, - model.toIdentifier(), - output, - startTime, - endTime); + predictionId, source, name, status, snapshotId, model, output, startTime, endTime); } } diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java index 6e8437cdeaf4..45af6f90f53b 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java @@ -67,7 +67,7 @@ void testRequestAndResponseWireFormat() throws Exception { .containsEntry("table", "clips"); assertThat((Map) requestMap.get("model")) .containsEntry("database", "quality") - .containsEntry("model", "qwen_vl"); + .containsEntry("object", "qwen_vl"); assertThat((Map) requestMap.get("options")) .containsEntry("scan.snapshot-id", "7"); @@ -139,8 +139,7 @@ void testStructuredModelPreservesDots() throws Exception { SubmitPredictionRequest decoded = RESTApi.fromJson(RESTApi.toJson(request), SubmitPredictionRequest.class); - assertThat(decoded.getModel().toIdentifier()) - .isEqualTo(Identifier.create("quality.archive", "qwen.vl")); + assertThat(decoded.getModel()).isEqualTo(Identifier.create("quality.archive", "qwen.vl")); } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java index b5481c90741e..3df656ec5b56 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java @@ -137,7 +137,7 @@ private static String predictionJson(String status) { + "\"table\":\"clips\"},\"name\":\"daily-quality\",\"status\":{\"state\":\"" + status + "\"},\"snapshotId\":7,\"model\":{\"database\":\"quality\"," - + "\"model\":\"qwen_vl\"}}"; + + "\"object\":\"qwen_vl\"}}"; } private static String terminatedPredictionJson(String code) { @@ -146,7 +146,7 @@ private static String terminatedPredictionJson(String code) { + "\"status\":{\"state\":\"TERMINATED\",\"terminationDetails\":{\"code\":\"" + code + "\"}},\"snapshotId\":7,\"model\":{\"database\":\"quality\"," - + "\"model\":\"qwen_vl\"}," + + "\"object\":\"qwen_vl\"}," + "\"output\":{\"table\":{" + "\"database\":\"system\",\"table\":\"results\"}}," + "\"startTime\":10,\"endTime\":20}"; diff --git a/paimon-python/pypaimon/api/prediction.py b/paimon-python/pypaimon/api/prediction.py index 267284ac6fe2..afb76fa708d9 100644 --- a/paimon-python/pypaimon/api/prediction.py +++ b/paimon-python/pypaimon/api/prediction.py @@ -52,24 +52,6 @@ def to_identifier(self) -> Identifier: return Identifier.create(self.database, self.table) -@dataclass -class PredictionModel: - database: str - model: str - - def __post_init__(self): - _nonblank(self.database, "database") - _nonblank(self.model, "model") - - @classmethod - def from_identifier(cls, identifier: Identifier) -> "PredictionModel": - return cls( - identifier.get_database_name(), identifier.get_object_name()) - - def to_identifier(self) -> Identifier: - return Identifier.create(self.database, self.model) - - @dataclass class PredictionOutput: table: Optional[PredictionSource] = optional_json_field( @@ -98,7 +80,7 @@ def to_catalog_output(self) -> CatalogPredictionOutput: @dataclass class SubmitPredictionRequest(RESTRequest): source: PredictionSource - model: PredictionModel + model: Identifier inputs: Dict[str, str] instructions: str merge_key_columns: Optional[List[str]] = optional_json_field( @@ -110,8 +92,8 @@ class SubmitPredictionRequest(RESTRequest): options: Dict[str, str] = field(default_factory=dict) def __post_init__(self): - if not isinstance(self.model, PredictionModel): - raise ValueError("model must be a PredictionModel") + if not isinstance(self.model, Identifier): + raise ValueError("model must be an Identifier") _nonblank(self.instructions, "instructions") for value, field_name in ( (self.name, "name"), @@ -129,7 +111,7 @@ def __post_init__(self): self.merge_key_columns, "merge_key_columns") self.options = _copy_string_mapping(self.options, "options", False) PredictionSpec( - model=self.model.to_identifier(), + model=self.model, inputs=self.inputs, instructions=self.instructions, merge_key_columns=self.merge_key_columns or [], @@ -145,7 +127,7 @@ def from_spec( return cls( source=PredictionSource.from_identifier(source), name=spec.name, - model=PredictionModel.from_identifier(spec.model), + model=spec.model, inputs=spec.inputs, instructions=spec.instructions, merge_key_columns=spec.merge_key_columns or None, @@ -167,7 +149,7 @@ class GetPredictionResponse(RESTResponse): source: PredictionSource = None status: PredictionStatus = None snapshot_id: int = json_field("snapshotId", default=0) - model: PredictionModel = None + model: Identifier = None name: Optional[str] = optional_json_field("name", "non_null") output: Optional[PredictionOutput] = optional_json_field( "output", "non_null") @@ -187,7 +169,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "GetPredictionResponse": status=PredictionStatus.from_dict(data.get("status")), snapshot_id=data.get("snapshotId"), model=( - PredictionModel(**data.get("model")) + Identifier(**data.get("model")) if isinstance(data.get("model"), dict) else data.get("model") ), @@ -205,7 +187,7 @@ def to_prediction(self) -> Prediction: name=self.name, status=self.status, snapshot_id=self.snapshot_id, - model=self.model.to_identifier(), + model=self.model, output=(self.output.to_catalog_output() if self.output is not None else None), start_time=self.start_time, diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index 31a5e4c41bcb..493b290580b5 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -27,7 +27,6 @@ GetPredictionResponse, ListPredictionsResponse, PredictionOutput as ApiPredictionOutput, - PredictionModel, PredictionSource, SubmitPredictionRequest, ) @@ -85,7 +84,7 @@ def _response(status=None, **changes): name="daily-quality", status=status, snapshot_id=7, - model=PredictionModel("quality", "qwen_vl"), + model=Identifier.create("quality", "qwen_vl"), ) values.update(changes) return GetPredictionResponse(**values) @@ -110,7 +109,7 @@ def test_request_and_response_wire_format(self): self.assertEqual( {"database": "quality", "table": "clips"}, body["source"]) self.assertEqual( - {"database": "quality", "model": "qwen_vl"}, body["model"]) + {"database": "quality", "object": "qwen_vl"}, body["model"]) self.assertNotIn("snapshotId", body) self.assertNotIn("tagName", body) @@ -230,7 +229,7 @@ def test_structured_model_preserves_dots(self): body = json.loads(JSON.to_json(request)) self.assertEqual( - {"database": "quality.archive", "model": "qwen.vl"}, + {"database": "quality.archive", "object": "qwen.vl"}, body["model"], ) From c4b618bcd65b01933d815fb2343205cdbce6a22f Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Tue, 17 Nov 2026 22:32:14 +0800 Subject: [PATCH 22/41] Complete prediction submission validation --- docs/static/rest-catalog-open-api.yaml | 10 ++++-- .../apache/paimon/catalog/PredictionSpec.java | 13 +++++++- .../requests/SubmitPredictionRequest.java | 11 +++++-- .../paimon/rest/RESTApiPredictionTest.java | 33 +++++++++++++++---- .../org/apache/paimon/rest/RESTCatalog.java | 2 ++ .../rest/RESTCatalogPredictionTest.java | 25 +++++++++++++- paimon-python/pypaimon/api/prediction.py | 1 + paimon-python/pypaimon/catalog/prediction.py | 4 +++ paimon-python/pypaimon/multimodal/table.py | 4 ++- .../pypaimon/tests/rest/prediction_test.py | 10 +++++- 10 files changed, 99 insertions(+), 14 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index b2df4fbf5d99..e59d3d365c79 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -139,7 +139,8 @@ paths: summary: Submit a prediction job operationId: submitPrediction description: >- - Accepts a job after fixing the source snapshot and resolving the registered model. + Records the source snapshot and resolves the registered model. Execution reads + the recorded snapshot and fails if it is no longer available. Reusing an idempotencyKey with the same payload returns the existing job; conflicting reuse returns 409. parameters: @@ -4731,7 +4732,12 @@ components: Optional key reused for retries of one logical submission. SDKs generate one by default; requests without a key are independent submissions. model: - $ref: '#/components/schemas/Identifier' + allOf: + - $ref: '#/components/schemas/Identifier' + required: [database, object] + properties: + database: {type: string, minLength: 1, maxLength: 128} + object: {type: string, minLength: 1, maxLength: 128} inputs: type: object minProperties: 1 diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java index 3893b11c1549..15426f5dec7b 100644 --- a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java @@ -47,6 +47,7 @@ public class PredictionSpec { private final String instructions; private final @Nullable String where; private final Map options; + private final @Nullable String idempotencyKey; public PredictionSpec( @Nullable String name, @@ -55,17 +56,22 @@ public PredictionSpec( @Nullable List mergeKeyColumns, String instructions, @Nullable String where, - @Nullable Map options) { + @Nullable Map options, + @Nullable String idempotencyKey) { checkOptionalNonBlank(name, "name"); checkArgument(model != null, "model must not be null"); + checkNonBlank(model.getDatabaseName(), "model database"); + checkNonBlank(model.getObjectName(), "model object"); checkNonBlank(instructions, "instructions"); checkOptionalNonBlank(where, "where"); + checkOptionalNonBlank(idempotencyKey, "idempotencyKey"); this.name = name; this.model = model; this.inputs = copyStringMap(inputs, "inputs", true); this.mergeKeyColumns = copyOptionalStringList(mergeKeyColumns, "mergeKeyColumns"); this.instructions = instructions; this.where = where; + this.idempotencyKey = idempotencyKey; this.options = options == null ? Collections.emptyMap() : copyStringMap(options, "options", false); CoreOptions coreOptions = new CoreOptions(this.options); @@ -107,6 +113,11 @@ public Map options() { return options; } + @Nullable + public String idempotencyKey() { + return idempotencyKey; + } + private static Map copyStringMap( Map values, String name, boolean nonEmpty) { checkArgument(values != null, "%s must not be null", name); diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java index 239efc7ffb2e..ca2404e9000e 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java @@ -76,14 +76,21 @@ public SubmitPredictionRequest( idempotencyKey == null ? UUID.randomUUID().toString() : idempotencyKey; this.spec = new PredictionSpec( - name, model, inputs, mergeKeyColumns, instructions, where, options); + name, + model, + inputs, + mergeKeyColumns, + instructions, + where, + options, + idempotencyKey); } public SubmitPredictionRequest(Identifier source, PredictionSpec spec) { this( new PredictionSource(source), spec.name(), - null, + spec.idempotencyKey(), spec.model(), spec.inputs(), spec.mergeKeyColumns(), diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java index 45af6f90f53b..34d738fad025 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java @@ -133,13 +133,15 @@ void testStructuredModelPreservesDots() throws Exception { null, "Classify", null, - Collections.emptyMap()); + Collections.emptyMap(), + "retry-1"); SubmitPredictionRequest request = new SubmitPredictionRequest(Identifier.create("quality", "clips"), spec); SubmitPredictionRequest decoded = RESTApi.fromJson(RESTApi.toJson(request), SubmitPredictionRequest.class); assertThat(decoded.getModel()).isEqualTo(Identifier.create("quality.archive", "qwen.vl")); + assertThat(decoded.getIdempotencyKey()).isEqualTo("retry-1"); } @Test @@ -153,7 +155,8 @@ void testSpecValidation() { null, "Classify", null, - Collections.emptyMap())) + Collections.emptyMap(), + null)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("inputs must not be empty"); @@ -166,7 +169,8 @@ void testSpecValidation() { Arrays.asList("clip_id", "clip_id"), "Classify", null, - Collections.emptyMap())) + Collections.emptyMap(), + null)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("mergeKeyColumns must not contain duplicates"); @@ -179,7 +183,8 @@ void testSpecValidation() { null, "Classify", null, - Collections.singletonMap("scan.snapshot-id", "0"))) + Collections.singletonMap("scan.snapshot-id", "0"), + null)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("scan.snapshot-id must be positive"); @@ -195,9 +200,24 @@ void testSpecValidation() { null, "Classify", null, - conflictingOptions)) + conflictingOptions, + null)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("mutually exclusive"); + + assertThatThrownBy( + () -> + new PredictionSpec( + null, + Identifier.create("", "qwen_vl"), + Collections.singletonMap("video", "content"), + null, + "Classify", + null, + Collections.emptyMap(), + null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("model database"); } @Test @@ -235,7 +255,8 @@ private static PredictionSpec spec() { Collections.singletonList("clip_id"), "Classify", null, - Collections.singletonMap("scan.snapshot-id", "7")); + Collections.singletonMap("scan.snapshot-id", "7"), + null); } private static Prediction prediction() { diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 72d3d25d04b1..69ace5880b3d 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -1217,6 +1217,8 @@ public void dropModel(Identifier identifier, boolean ignoreIfNotExists) @Override public Prediction submitPrediction(Identifier identifier, PredictionSpec spec) throws TableNotExistException, ModelNotExistException { + checkNotBranch(identifier, "submitPrediction"); + checkNotSystemTable(identifier, "submitPrediction"); try { return api.submitPrediction(new SubmitPredictionRequest(identifier, spec)) .toPrediction(); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java index 3df656ec5b56..fd5c24dc09c8 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java @@ -112,6 +112,28 @@ void testPredictionNotFoundPreservesResourceType() throws Exception { } } + @Test + void testPredictionRejectsBranchAndSystemTable() throws Exception { + try (MockWebServer server = new MockWebServer()) { + server.start(); + Catalog catalog = catalog(server); + + assertThatThrownBy( + () -> + catalog.submitPrediction( + new Identifier("quality", "clips", "dev"), spec())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("branch"); + assertThatThrownBy( + () -> + catalog.submitPrediction( + new Identifier("quality", "clips", null, "snapshots"), + spec())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("system table"); + } + } + private static PredictionSpec spec() { return new PredictionSpec( "daily-quality", @@ -120,7 +142,8 @@ private static PredictionSpec spec() { null, "Classify", null, - Collections.emptyMap()); + Collections.emptyMap(), + null); } private static Catalog catalog(MockWebServer server) { diff --git a/paimon-python/pypaimon/api/prediction.py b/paimon-python/pypaimon/api/prediction.py index afb76fa708d9..c6a816377ef9 100644 --- a/paimon-python/pypaimon/api/prediction.py +++ b/paimon-python/pypaimon/api/prediction.py @@ -131,6 +131,7 @@ def from_spec( inputs=spec.inputs, instructions=spec.instructions, merge_key_columns=spec.merge_key_columns or None, + idempotency_key=spec.idempotency_key, where=spec.where, options=spec.options, ) diff --git a/paimon-python/pypaimon/catalog/prediction.py b/paimon-python/pypaimon/catalog/prediction.py index 366e49991224..8e19e2861c19 100644 --- a/paimon-python/pypaimon/catalog/prediction.py +++ b/paimon-python/pypaimon/catalog/prediction.py @@ -115,16 +115,20 @@ class PredictionSpec: name: Optional[str] = None where: Optional[str] = None options: Dict[str, str] = field(default_factory=dict) + idempotency_key: Optional[str] = None def __post_init__(self): if isinstance(self.model, str): self.model = Identifier.from_string(self.model) if not isinstance(self.model, Identifier): raise ValueError("model must be an Identifier or qualified name") + _nonblank(self.model.get_database_name(), "model database") + _nonblank(self.model.get_object_name(), "model object") _nonblank(self.instructions, "instructions") for value, name in ( (self.name, "name"), (self.where, "where"), + (self.idempotency_key, "idempotency_key"), ): if value is not None: _nonblank(value, name) diff --git a/paimon-python/pypaimon/multimodal/table.py b/paimon-python/pypaimon/multimodal/table.py index 923a69f54a5d..e420bd877bda 100644 --- a/paimon-python/pypaimon/multimodal/table.py +++ b/paimon-python/pypaimon/multimodal/table.py @@ -120,7 +120,8 @@ def predict( merge_key_columns=None, name=None, where=None, - options=None): + options=None, + idempotency_key=None): """Submit an asynchronous prediction without reading rows locally. ``options`` uses Paimon dynamic table option names. Time travel uses @@ -173,6 +174,7 @@ def predict( [] if merge_key_columns is None else merge_key_columns), where=where, options=prediction_options, + idempotency_key=idempotency_key, ) prediction = self.catalog.submit_prediction( self.raw_table.identifier, spec) diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index 493b290580b5..665c4ee4cce4 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -124,10 +124,14 @@ def test_request_and_response_wire_format(self): keyed = SubmitPredictionRequest.from_spec( Identifier.create("quality", "clips"), - _spec(merge_key_columns=["clip_id"]), + _spec( + merge_key_columns=["clip_id"], + idempotency_key="retry-1", + ), ) keyed_body = json.loads(JSON.to_json(keyed)) self.assertEqual(["clip_id"], keyed_body["mergeKeyColumns"]) + self.assertEqual("retry-1", keyed_body["idempotencyKey"]) response = JSON.from_json( JSON.to_json(_response()), GetPredictionResponse) @@ -212,6 +216,8 @@ def test_validation(self): for changes in invalid: with self.subTest(changes=changes), self.assertRaises(ValueError): _spec(**changes) + with self.assertRaises(ValueError): + _spec(model=Identifier.create("", "qwen_vl")) with self.assertRaises(ValueError): PredictionStatus(PredictionState.TERMINATED) with self.assertRaises(ValueError): @@ -246,6 +252,7 @@ def test_table_submit_fixes_view_and_returns_waiter(self): model="quality.qwen_vl", inputs={"video": "content"}, instructions="Classify", + idempotency_key="retry-1", ) self.assertIsInstance(waiter, PredictionWait) @@ -253,6 +260,7 @@ def test_table_submit_fixes_view_and_returns_waiter(self): sent = catalog.submit_prediction.call_args[0][1] self.assertEqual({"scan.snapshot-id": "7"}, sent.options) self.assertEqual([], sent.merge_key_columns) + self.assertEqual("retry-1", sent.idempotency_key) raw.new_read_builder.assert_not_called() def test_table_queries_return_metadata_and_cancel_returns_waiter(self): From f0e8d227eb399cb134888909b7ea32fa81974343 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Wed, 18 Nov 2026 00:32:14 +0800 Subject: [PATCH 23/41] Align prediction idempotency tokens --- docs/static/rest-catalog-open-api.yaml | 10 +++---- .../apache/paimon/catalog/PredictionSpec.java | 15 ++++++----- .../requests/SubmitPredictionRequest.java | 27 ++++++++++--------- .../paimon/rest/RESTApiPredictionTest.java | 24 +++++++++++++---- .../rest/RESTCatalogPredictionTest.java | 4 +-- paimon-python/pypaimon/api/prediction.py | 15 ++++++----- paimon-python/pypaimon/catalog/prediction.py | 8 ++++-- paimon-python/pypaimon/multimodal/table.py | 4 +-- .../pypaimon/tests/rest/prediction_test.py | 17 ++++++------ 9 files changed, 76 insertions(+), 48 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index e59d3d365c79..fd8ee75f5ed3 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -141,7 +141,7 @@ paths: description: >- Records the source snapshot and resolves the registered model. Execution reads the recorded snapshot and fails if it is no longer available. - Reusing an idempotencyKey with the same payload returns the existing job; + Reusing an idempotencyToken with the same payload returns the existing job; conflicting reuse returns 409. parameters: - {name: prefix, in: path, required: true, schema: {type: string}} @@ -4724,13 +4724,13 @@ components: minLength: 1 maxLength: 128 description: Optional non-unique label. - idempotencyKey: + idempotencyToken: type: string minLength: 1 - maxLength: 128 + maxLength: 64 description: >- - Optional key reused for retries of one logical submission. SDKs generate - one by default; requests without a key are independent submissions. + Optional token reused for retries of one logical submission. SDKs generate + one by default; requests without a token are independent submissions. model: allOf: - $ref: '#/components/schemas/Identifier' diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java index 15426f5dec7b..d0f98313f288 100644 --- a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java @@ -47,7 +47,7 @@ public class PredictionSpec { private final String instructions; private final @Nullable String where; private final Map options; - private final @Nullable String idempotencyKey; + private final @Nullable String idempotencyToken; public PredictionSpec( @Nullable String name, @@ -57,21 +57,24 @@ public PredictionSpec( String instructions, @Nullable String where, @Nullable Map options, - @Nullable String idempotencyKey) { + @Nullable String idempotencyToken) { checkOptionalNonBlank(name, "name"); checkArgument(model != null, "model must not be null"); checkNonBlank(model.getDatabaseName(), "model database"); checkNonBlank(model.getObjectName(), "model object"); checkNonBlank(instructions, "instructions"); checkOptionalNonBlank(where, "where"); - checkOptionalNonBlank(idempotencyKey, "idempotencyKey"); + checkOptionalNonBlank(idempotencyToken, "idempotencyToken"); + checkArgument( + idempotencyToken == null || idempotencyToken.length() <= 64, + "idempotencyToken must not exceed 64 characters"); this.name = name; this.model = model; this.inputs = copyStringMap(inputs, "inputs", true); this.mergeKeyColumns = copyOptionalStringList(mergeKeyColumns, "mergeKeyColumns"); this.instructions = instructions; this.where = where; - this.idempotencyKey = idempotencyKey; + this.idempotencyToken = idempotencyToken; this.options = options == null ? Collections.emptyMap() : copyStringMap(options, "options", false); CoreOptions coreOptions = new CoreOptions(this.options); @@ -114,8 +117,8 @@ public Map options() { } @Nullable - public String idempotencyKey() { - return idempotencyKey; + public String idempotencyToken() { + return idempotencyToken; } private static Map copyStringMap( diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java index ca2404e9000e..44d5b97b3f00 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java @@ -43,7 +43,7 @@ public class SubmitPredictionRequest implements RESTRequest { private static final String FIELD_NAME = "name"; private static final String FIELD_SOURCE = "source"; - private static final String FIELD_IDEMPOTENCY_KEY = "idempotencyKey"; + private static final String FIELD_IDEMPOTENCY_TOKEN = "idempotencyToken"; private static final String FIELD_MODEL = "model"; private static final String FIELD_INPUTS = "inputs"; private static final String FIELD_MERGE_KEY_COLUMNS = "mergeKeyColumns"; @@ -51,7 +51,7 @@ public class SubmitPredictionRequest implements RESTRequest { private static final String FIELD_WHERE = "where"; private static final String FIELD_OPTIONS = "options"; - private final String idempotencyKey; + private final String idempotencyToken; private final PredictionSource source; private final PredictionSpec spec; @@ -59,7 +59,7 @@ public class SubmitPredictionRequest implements RESTRequest { public SubmitPredictionRequest( @JsonProperty(FIELD_SOURCE) PredictionSource source, @Nullable @JsonProperty(FIELD_NAME) String name, - @Nullable @JsonProperty(FIELD_IDEMPOTENCY_KEY) String idempotencyKey, + @Nullable @JsonProperty(FIELD_IDEMPOTENCY_TOKEN) String idempotencyToken, @JsonProperty(FIELD_MODEL) Identifier model, @JsonProperty(FIELD_INPUTS) Map inputs, @Nullable @JsonProperty(FIELD_MERGE_KEY_COLUMNS) List mergeKeyColumns, @@ -69,11 +69,14 @@ public SubmitPredictionRequest( checkArgument(source != null, "source must not be null"); checkArgument(model != null, "model must not be null"); checkArgument( - idempotencyKey == null || !idempotencyKey.trim().isEmpty(), - "idempotencyKey must not be blank"); + idempotencyToken == null || !idempotencyToken.trim().isEmpty(), + "idempotencyToken must not be blank"); + checkArgument( + idempotencyToken == null || idempotencyToken.length() <= 64, + "idempotencyToken must not exceed 64 characters"); this.source = source; - this.idempotencyKey = - idempotencyKey == null ? UUID.randomUUID().toString() : idempotencyKey; + this.idempotencyToken = + idempotencyToken == null ? UUID.randomUUID().toString() : idempotencyToken; this.spec = new PredictionSpec( name, @@ -83,14 +86,14 @@ public SubmitPredictionRequest( instructions, where, options, - idempotencyKey); + idempotencyToken); } public SubmitPredictionRequest(Identifier source, PredictionSpec spec) { this( new PredictionSource(source), spec.name(), - spec.idempotencyKey(), + spec.idempotencyToken(), spec.model(), spec.inputs(), spec.mergeKeyColumns(), @@ -111,9 +114,9 @@ public String getName() { return spec.name(); } - @JsonGetter(FIELD_IDEMPOTENCY_KEY) - public String getIdempotencyKey() { - return idempotencyKey; + @JsonGetter(FIELD_IDEMPOTENCY_TOKEN) + public String getIdempotencyToken() { + return idempotencyToken; } @JsonGetter(FIELD_MODEL) diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java index 34d738fad025..bfd69790a595 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java @@ -46,10 +46,10 @@ void testRequestAndResponseWireFormat() throws Exception { SubmitPredictionRequest request = new SubmitPredictionRequest(Identifier.create("quality", "clips"), spec()); Map requestMap = RESTApi.fromJson(RESTApi.toJson(request), Map.class); - assertThat(requestMap.get("idempotencyKey")).isInstanceOf(String.class); - assertThat((String) requestMap.get("idempotencyKey")).isNotBlank(); - assertThat(RESTApi.fromJson(RESTApi.toJson(request), Map.class).get("idempotencyKey")) - .isEqualTo(requestMap.get("idempotencyKey")); + assertThat(requestMap.get("idempotencyToken")).isInstanceOf(String.class); + assertThat((String) requestMap.get("idempotencyToken")).isNotBlank(); + assertThat(RESTApi.fromJson(RESTApi.toJson(request), Map.class).get("idempotencyToken")) + .isEqualTo(requestMap.get("idempotencyToken")); assertThat(requestMap) .containsEntry("instructions", "Classify") .doesNotContainKeys( @@ -141,7 +141,7 @@ void testStructuredModelPreservesDots() throws Exception { RESTApi.fromJson(RESTApi.toJson(request), SubmitPredictionRequest.class); assertThat(decoded.getModel()).isEqualTo(Identifier.create("quality.archive", "qwen.vl")); - assertThat(decoded.getIdempotencyKey()).isEqualTo("retry-1"); + assertThat(decoded.getIdempotencyToken()).isEqualTo("retry-1"); } @Test @@ -218,6 +218,20 @@ void testSpecValidation() { null)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("model database"); + + assertThatThrownBy( + () -> + new PredictionSpec( + null, + Identifier.create("quality", "qwen_vl"), + Collections.singletonMap("video", "content"), + null, + "Classify", + null, + Collections.emptyMap(), + String.join("", Collections.nCopies(65, "x")))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("64 characters"); } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java index fd5c24dc09c8..5ceb14856343 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java @@ -55,8 +55,8 @@ void testPredictionLifecycleUsesSharedRestContract() throws Exception { RecordedRequest submit = server.takeRequest(10, TimeUnit.SECONDS); assertThat(submit.getPath()).isEqualTo("/v1/catalog/predictions"); Map request = RESTApi.fromJson(submit.getBody().readUtf8(), Map.class); - assertThat(request.get("idempotencyKey")).isInstanceOf(String.class); - assertThat((String) request.get("idempotencyKey")).isNotBlank(); + assertThat(request.get("idempotencyToken")).isInstanceOf(String.class); + assertThat((String) request.get("idempotencyToken")).isNotBlank(); assertThat(request) .doesNotContainKeys("resultColumns", "mergeKeyColumns", "outputSchema"); assertThat((Map) request.get("source")) diff --git a/paimon-python/pypaimon/api/prediction.py b/paimon-python/pypaimon/api/prediction.py index c6a816377ef9..abe13ed95f38 100644 --- a/paimon-python/pypaimon/api/prediction.py +++ b/paimon-python/pypaimon/api/prediction.py @@ -86,8 +86,8 @@ class SubmitPredictionRequest(RESTRequest): merge_key_columns: Optional[List[str]] = optional_json_field( "mergeKeyColumns", "non_null") name: Optional[str] = optional_json_field("name", "non_null") - idempotency_key: Optional[str] = optional_json_field( - "idempotencyKey", "non_null") + idempotency_token: Optional[str] = optional_json_field( + "idempotencyToken", "non_null") where: Optional[str] = optional_json_field("where", "non_null") options: Dict[str, str] = field(default_factory=dict) @@ -101,10 +101,13 @@ def __post_init__(self): ): if value is not None: _nonblank(value, field_name) - if self.idempotency_key is None: - self.idempotency_key = uuid.uuid4().hex + if self.idempotency_token is None: + self.idempotency_token = uuid.uuid4().hex else: - _nonblank(self.idempotency_key, "idempotency_key") + _nonblank(self.idempotency_token, "idempotency_token") + if len(self.idempotency_token) > 64: + raise ValueError( + "idempotency_token must not exceed 64 characters") self.inputs = _copy_string_mapping(self.inputs, "inputs", True) if self.merge_key_columns is not None: self.merge_key_columns = _copy_string_list( @@ -131,7 +134,7 @@ def from_spec( inputs=spec.inputs, instructions=spec.instructions, merge_key_columns=spec.merge_key_columns or None, - idempotency_key=spec.idempotency_key, + idempotency_token=spec.idempotency_token, where=spec.where, options=spec.options, ) diff --git a/paimon-python/pypaimon/catalog/prediction.py b/paimon-python/pypaimon/catalog/prediction.py index 8e19e2861c19..f3c6913b3865 100644 --- a/paimon-python/pypaimon/catalog/prediction.py +++ b/paimon-python/pypaimon/catalog/prediction.py @@ -115,7 +115,7 @@ class PredictionSpec: name: Optional[str] = None where: Optional[str] = None options: Dict[str, str] = field(default_factory=dict) - idempotency_key: Optional[str] = None + idempotency_token: Optional[str] = None def __post_init__(self): if isinstance(self.model, str): @@ -128,10 +128,14 @@ def __post_init__(self): for value, name in ( (self.name, "name"), (self.where, "where"), - (self.idempotency_key, "idempotency_key"), + (self.idempotency_token, "idempotency_token"), ): if value is not None: _nonblank(value, name) + if (self.idempotency_token is not None + and len(self.idempotency_token) > 64): + raise ValueError( + "idempotency_token must not exceed 64 characters") self.inputs = _copy_string_mapping(self.inputs, "inputs", True) self.merge_key_columns = _copy_string_list( self.merge_key_columns, "merge_key_columns") diff --git a/paimon-python/pypaimon/multimodal/table.py b/paimon-python/pypaimon/multimodal/table.py index e420bd877bda..801500f6f0cf 100644 --- a/paimon-python/pypaimon/multimodal/table.py +++ b/paimon-python/pypaimon/multimodal/table.py @@ -121,7 +121,7 @@ def predict( name=None, where=None, options=None, - idempotency_key=None): + idempotency_token=None): """Submit an asynchronous prediction without reading rows locally. ``options`` uses Paimon dynamic table option names. Time travel uses @@ -174,7 +174,7 @@ def predict( [] if merge_key_columns is None else merge_key_columns), where=where, options=prediction_options, - idempotency_key=idempotency_key, + idempotency_token=idempotency_token, ) prediction = self.catalog.submit_prediction( self.raw_table.identifier, spec) diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index 665c4ee4cce4..0027c8b46dca 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -96,10 +96,10 @@ def test_request_and_response_wire_format(self): request = SubmitPredictionRequest.from_spec( Identifier.create("quality", "clips"), _spec()) body = json.loads(JSON.to_json(request)) - self.assertTrue(body["idempotencyKey"]) + self.assertTrue(body["idempotencyToken"]) self.assertEqual( - body["idempotencyKey"], - json.loads(JSON.to_json(request))["idempotencyKey"], + body["idempotencyToken"], + json.loads(JSON.to_json(request))["idempotencyToken"], ) self.assertNotIn("resultColumns", body) self.assertNotIn("mergeKeyColumns", body) @@ -126,12 +126,12 @@ def test_request_and_response_wire_format(self): Identifier.create("quality", "clips"), _spec( merge_key_columns=["clip_id"], - idempotency_key="retry-1", + idempotency_token="retry-1", ), ) keyed_body = json.loads(JSON.to_json(keyed)) self.assertEqual(["clip_id"], keyed_body["mergeKeyColumns"]) - self.assertEqual("retry-1", keyed_body["idempotencyKey"]) + self.assertEqual("retry-1", keyed_body["idempotencyToken"]) response = JSON.from_json( JSON.to_json(_response()), GetPredictionResponse) @@ -212,6 +212,7 @@ def test_validation(self): "scan.tag-name": "v1", }), dict(options={"parallelism": 2}), + dict(idempotency_token="x" * 65), ] for changes in invalid: with self.subTest(changes=changes), self.assertRaises(ValueError): @@ -252,7 +253,7 @@ def test_table_submit_fixes_view_and_returns_waiter(self): model="quality.qwen_vl", inputs={"video": "content"}, instructions="Classify", - idempotency_key="retry-1", + idempotency_token="retry-1", ) self.assertIsInstance(waiter, PredictionWait) @@ -260,7 +261,7 @@ def test_table_submit_fixes_view_and_returns_waiter(self): sent = catalog.submit_prediction.call_args[0][1] self.assertEqual({"scan.snapshot-id": "7"}, sent.options) self.assertEqual([], sent.merge_key_columns) - self.assertEqual("retry-1", sent.idempotency_key) + self.assertEqual("retry-1", sent.idempotency_token) raw.new_read_builder.assert_not_called() def test_table_queries_return_metadata_and_cancel_returns_waiter(self): @@ -399,7 +400,7 @@ def do_GET(self): {"database": "a/b", "table": "clip table"}, calls[-1][2]["source"], ) - self.assertTrue(calls[-1][2]["idempotencyKey"]) + self.assertTrue(calls[-1][2]["idempotencyToken"]) self.assertNotIn("mergeKeyColumns", calls[-1][2]) page = catalog.list_predictions( From 5344355241762b1402d0626036005b86f69a6b4d Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Wed, 18 Nov 2026 12:17:42 +0800 Subject: [PATCH 24/41] Fix prediction REST coverage fixture --- paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py b/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py index c63a597a95b8..973f9e86fd31 100644 --- a/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py +++ b/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py @@ -71,7 +71,7 @@ ), SubmitPredictionRequest: SubmitPredictionRequest( source=PredictionSource("db", "t"), - model="db.qwen_vl", + model=Identifier.create("db", "qwen_vl"), inputs={"video": "content"}, instructions="classify", ), From 1760f696c440d9d0209bf60386121f4337458af3 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Wed, 18 Nov 2026 12:28:21 +0800 Subject: [PATCH 25/41] Clarify prediction lifecycle semantics --- docs/static/rest-catalog-open-api.yaml | 26 +++++++++++++-- .../org/apache/paimon/catalog/Prediction.java | 2 ++ .../paimon/catalog/PredictionStatus.java | 10 ++++-- .../rest/responses/GetPredictionResponse.java | 2 ++ .../paimon/rest/RESTApiPredictionTest.java | 33 +++++++++++-------- paimon-python/pypaimon/api/prediction.py | 2 ++ paimon-python/pypaimon/catalog/prediction.py | 21 ++++++++---- .../pypaimon/tests/rest/prediction_test.py | 15 +++++++++ 8 files changed, 84 insertions(+), 27 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index fd8ee75f5ed3..acbfdb12cbb6 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -4694,7 +4694,19 @@ components: $ref: '#/components/schemas/PredictionState' terminationDetails: $ref: '#/components/schemas/PredictionTerminationDetails' - description: terminationDetails is present when state is TERMINATING or TERMINATED. + oneOf: + - properties: + state: {enum: [PENDING, RUNNING]} + not: + required: [terminationDetails] + - properties: + state: {const: TERMINATING} + - properties: + state: {const: TERMINATED} + required: [terminationDetails] + description: >- + terminationDetails is forbidden for PENDING and RUNNING, optional for + TERMINATING, and required for TERMINATED. PredictionSource: type: object additionalProperties: false @@ -4790,8 +4802,16 @@ components: $ref: '#/components/schemas/Identifier' output: $ref: '#/components/schemas/PredictionOutput' - startTime: {type: integer, format: int64, minimum: 0} - endTime: {type: integer, format: int64, minimum: 0} + startTime: + type: integer + format: int64 + minimum: 0 + description: Start time as Unix epoch milliseconds in UTC. + endTime: + type: integer + format: int64 + minimum: 0 + description: End time as Unix epoch milliseconds in UTC. ListPredictionsResponse: type: object required: [predictions] diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java index 53a42e5eae18..5601a02414aa 100644 --- a/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java @@ -96,11 +96,13 @@ public PredictionOutput output() { return output; } + /** Start time as Unix epoch milliseconds in UTC, or null if not started. */ @Nullable public Long startTime() { return startTime; } + /** End time as Unix epoch milliseconds in UTC, or null if not terminated. */ @Nullable public Long endTime() { return endTime; diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionStatus.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionStatus.java index bf90ad9a2f81..527004d191ba 100644 --- a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionStatus.java +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionStatus.java @@ -88,10 +88,14 @@ public PredictionStatus( @JsonProperty("state") State state, @Nullable @JsonProperty("terminationDetails") TerminationDetails terminationDetails) { checkArgument(state != null, "state must not be null"); - boolean terminating = state == State.TERMINATING || state == State.TERMINATED; checkArgument( - terminating == (terminationDetails != null), - "terminationDetails must be present only for terminating states"); + state != State.TERMINATED || terminationDetails != null, + "terminationDetails must be present for TERMINATED state"); + checkArgument( + state == State.TERMINATING + || state == State.TERMINATED + || terminationDetails == null, + "terminationDetails must not be present before termination starts"); this.state = state; this.terminationDetails = terminationDetails; } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java index a0661fc814ab..c91054463443 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java @@ -121,6 +121,7 @@ public PredictionOutput getOutput() { return output; } + /** Start time as Unix epoch milliseconds in UTC, or null if not started. */ @Nullable @JsonInclude(JsonInclude.Include.NON_NULL) @JsonGetter("startTime") @@ -128,6 +129,7 @@ public Long getStartTime() { return startTime; } + /** End time as Unix epoch milliseconds in UTC, or null if not terminated. */ @Nullable @JsonInclude(JsonInclude.Include.NON_NULL) @JsonGetter("endTime") diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java index bfd69790a595..e93da7c8d224 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java @@ -240,23 +240,28 @@ void testTerminalStates() { .isFalse(); assertThat(new PredictionStatus(PredictionStatus.State.RUNNING, null).isTerminal()) .isFalse(); - assertThat( - new PredictionStatus( - PredictionStatus.State.TERMINATING, - new PredictionStatus.TerminationDetails( - PredictionStatus.TerminationCode.CANCELED, null)) - .isTerminal()) + assertThat(new PredictionStatus(PredictionStatus.State.TERMINATING, null).isTerminal()) .isFalse(); - PredictionStatus terminated = - new PredictionStatus( - PredictionStatus.State.TERMINATED, - new PredictionStatus.TerminationDetails( - PredictionStatus.TerminationCode.SUCCESS, null)); - assertThat(terminated.isTerminal()).isTrue(); - assertThat(terminated.terminationDetails().code()) - .isEqualTo(PredictionStatus.TerminationCode.SUCCESS); + for (PredictionStatus.TerminationCode code : + Arrays.asList( + PredictionStatus.TerminationCode.CANCELED, + PredictionStatus.TerminationCode.SUCCESS)) { + PredictionStatus terminated = + new PredictionStatus( + PredictionStatus.State.TERMINATED, + new PredictionStatus.TerminationDetails(code, null)); + assertThat(terminated.isTerminal()).isTrue(); + assertThat(terminated.terminationDetails().code()).isEqualTo(code); + } assertThatThrownBy(() -> new PredictionStatus(PredictionStatus.State.TERMINATED, null)) .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy( + () -> + new PredictionStatus( + PredictionStatus.State.RUNNING, + new PredictionStatus.TerminationDetails( + PredictionStatus.TerminationCode.SUCCESS, null))) + .isInstanceOf(IllegalArgumentException.class); } private static PredictionSpec spec() { diff --git a/paimon-python/pypaimon/api/prediction.py b/paimon-python/pypaimon/api/prediction.py index abe13ed95f38..f050222d917c 100644 --- a/paimon-python/pypaimon/api/prediction.py +++ b/paimon-python/pypaimon/api/prediction.py @@ -149,6 +149,8 @@ class CancelPredictionRequest(RESTRequest): @dataclass class GetPredictionResponse(RESTResponse): + """Prediction metadata with UTC Unix epoch millisecond timestamps.""" + prediction_id: str = json_field("predictionId") source: PredictionSource = None status: PredictionStatus = None diff --git a/paimon-python/pypaimon/catalog/prediction.py b/paimon-python/pypaimon/catalog/prediction.py index f3c6913b3865..9f2d93bf0b0e 100644 --- a/paimon-python/pypaimon/catalog/prediction.py +++ b/paimon-python/pypaimon/catalog/prediction.py @@ -65,14 +65,18 @@ def __post_init__(self): self.state = PredictionState(self.state) if not isinstance(self.state, PredictionState): raise ValueError("state must be a PredictionState") - terminating = self.state in { - PredictionState.TERMINATING, - PredictionState.TERMINATED, + if (self.state is PredictionState.TERMINATED + and self.termination_details is None): + raise ValueError( + "termination_details must be present for TERMINATED state") + accepts_details = self.state in { + PredictionState.TERMINATING, + PredictionState.TERMINATED, } - if terminating != (self.termination_details is not None): + if not accepts_details and self.termination_details is not None: raise ValueError( - "termination_details must be present only for " - "terminating states") + "termination_details must not be present before " + "termination starts") if self.termination_details is not None and not isinstance( self.termination_details, PredictionTerminationDetails): raise ValueError( @@ -167,7 +171,10 @@ def __post_init__(self): @dataclass class Prediction: - """Durable metadata for one asynchronous prediction job.""" + """Durable metadata for one asynchronous prediction job. + + ``start_time`` and ``end_time`` are Unix epoch milliseconds in UTC. + """ prediction_id: str source: Identifier diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index 0027c8b46dca..da795f749481 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -228,6 +228,21 @@ def test_validation(self): PredictionTerminationCode.FAILED), ) + def test_terminating_does_not_require_final_result(self): + terminating = PredictionStatus(PredictionState.TERMINATING) + self.assertFalse(terminating.terminal) + for code in ( + PredictionTerminationCode.CANCELED, + PredictionTerminationCode.SUCCESS, + ): + with self.subTest(code=code): + terminated = PredictionStatus( + PredictionState.TERMINATED, + PredictionTerminationDetails(code), + ) + self.assertTrue(terminated.terminal) + self.assertEqual(code, terminated.termination_details.code) + def test_structured_model_preserves_dots(self): request = SubmitPredictionRequest.from_spec( Identifier.create("quality", "clips"), From ed98faebc8dce2942df9b7697bc809998a724107 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Wed, 18 Nov 2026 13:55:07 +0800 Subject: [PATCH 26/41] Fix prediction status indentation --- paimon-python/pypaimon/catalog/prediction.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paimon-python/pypaimon/catalog/prediction.py b/paimon-python/pypaimon/catalog/prediction.py index 9f2d93bf0b0e..04d9fe66b857 100644 --- a/paimon-python/pypaimon/catalog/prediction.py +++ b/paimon-python/pypaimon/catalog/prediction.py @@ -70,8 +70,8 @@ def __post_init__(self): raise ValueError( "termination_details must be present for TERMINATED state") accepts_details = self.state in { - PredictionState.TERMINATING, - PredictionState.TERMINATED, + PredictionState.TERMINATING, + PredictionState.TERMINATED, } if not accepts_details and self.termination_details is not None: raise ValueError( From a099fdc0acaa34af3322c46e1d938203ec100d87 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Wed, 18 Nov 2026 14:46:29 +0800 Subject: [PATCH 27/41] Restrict prediction output to tables --- docs/static/rest-catalog-open-api.yaml | 4 ++-- .../org/apache/paimon/catalog/PredictionOutput.java | 10 ++++------ .../org/apache/paimon/rest/RESTApiPredictionTest.java | 3 +++ paimon-python/pypaimon/api/prediction.py | 11 +++-------- paimon-python/pypaimon/catalog/prediction.py | 6 +++--- paimon-python/pypaimon/tests/rest/prediction_test.py | 3 +++ 6 files changed, 18 insertions(+), 19 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index acbfdb12cbb6..3c022a915415 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -4716,14 +4716,14 @@ components: table: {type: string, minLength: 1, maxLength: 128} PredictionOutput: type: object - minProperties: 1 + additionalProperties: false + required: [table] properties: table: $ref: '#/components/schemas/PredictionSource' description: >- Managed result table containing source identity columns, a response string, and an error string. - additionalProperties: true SubmitPredictionRequest: type: object additionalProperties: false diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionOutput.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionOutput.java index 0097df3d26a4..2ea3d165036e 100644 --- a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionOutput.java +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionOutput.java @@ -23,25 +23,23 @@ import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; -import javax.annotation.Nullable; +import static org.apache.paimon.utils.Preconditions.checkArgument; /** Output of an asynchronous prediction. */ @Experimental @JsonIgnoreProperties(ignoreUnknown = true) public class PredictionOutput { - private final @Nullable PredictionSource table; + private final PredictionSource table; @JsonCreator - public PredictionOutput(@Nullable @JsonProperty("table") PredictionSource table) { + public PredictionOutput(@JsonProperty("table") PredictionSource table) { + checkArgument(table != null, "table must not be null"); this.table = table; } - @Nullable - @JsonInclude(JsonInclude.Include.NON_NULL) @JsonGetter("table") public PredictionSource table() { return table; diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java index e93da7c8d224..941d224efa9f 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java @@ -115,6 +115,9 @@ void testRequestAndResponseWireFormat() throws Exception { assertThat((Map) output.get("table")) .containsEntry("database", "system") .containsEntry("table", "prediction_results"); + assertThatThrownBy(() -> new PredictionOutput(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("table"); ListPredictionsResponse empty = RESTApi.fromJson("{}", ListPredictionsResponse.class); assertThat(empty.getPredictions()).isEmpty(); diff --git a/paimon-python/pypaimon/api/prediction.py b/paimon-python/pypaimon/api/prediction.py index f050222d917c..036933677770 100644 --- a/paimon-python/pypaimon/api/prediction.py +++ b/paimon-python/pypaimon/api/prediction.py @@ -54,12 +54,10 @@ def to_identifier(self) -> Identifier: @dataclass class PredictionOutput: - table: Optional[PredictionSource] = optional_json_field( - "table", "non_null") + table: PredictionSource def __post_init__(self): - if self.table is not None and not isinstance( - self.table, PredictionSource): + if not isinstance(self.table, PredictionSource): raise ValueError("table must be a PredictionSource") @classmethod @@ -71,10 +69,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "PredictionOutput": ) def to_catalog_output(self) -> CatalogPredictionOutput: - return CatalogPredictionOutput( - table=(self.table.to_identifier() - if self.table is not None else None), - ) + return CatalogPredictionOutput(table=self.table.to_identifier()) @dataclass diff --git a/paimon-python/pypaimon/catalog/prediction.py b/paimon-python/pypaimon/catalog/prediction.py index 04d9fe66b857..1d86206a44e7 100644 --- a/paimon-python/pypaimon/catalog/prediction.py +++ b/paimon-python/pypaimon/catalog/prediction.py @@ -160,12 +160,12 @@ def __post_init__(self): @dataclass class PredictionOutput: - """Known output locations of an asynchronous prediction.""" + """Managed result table of an asynchronous prediction.""" - table: Optional[Identifier] = None + table: Identifier def __post_init__(self): - if self.table is not None and not isinstance(self.table, Identifier): + if not isinstance(self.table, Identifier): raise ValueError("table must be an Identifier") diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index da795f749481..7ab325611c85 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -191,6 +191,9 @@ def test_request_and_response_wire_format(self): Identifier.create("system", "prediction_results")), completed.to_prediction().output, ) + with self.assertRaises(ValueError): + ApiPredictionOutput.from_dict( + {"artifactUri": "oss://bucket/result"}) page = JSON.from_json( JSON.to_json(ListPredictionsResponse([_response()], "next")), ListPredictionsResponse, From 85aeefda6ae411a674df0a4d20015f9e938d3b37 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Wed, 18 Nov 2026 15:40:39 +0800 Subject: [PATCH 28/41] Harden prediction wait and error handling --- .../requests/SubmitPredictionRequest.java | 2 +- .../paimon/rest/RESTApiPredictionTest.java | 2 + .../org/apache/paimon/rest/RESTCatalog.java | 15 +++++-- .../rest/RESTCatalogPredictionTest.java | 21 +++++++++ .../pypaimon/catalog/rest/rest_catalog.py | 12 +++-- paimon-python/pypaimon/common/wait.py | 5 ++- .../pypaimon/multimodal/prediction_wait.py | 2 + .../pypaimon/tests/rest/prediction_test.py | 44 +++++++++++++++++++ 8 files changed, 95 insertions(+), 8 deletions(-) diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java index 44d5b97b3f00..737de6492a71 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java @@ -86,7 +86,7 @@ public SubmitPredictionRequest( instructions, where, options, - idempotencyToken); + this.idempotencyToken); } public SubmitPredictionRequest(Identifier source, PredictionSpec spec) { diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java index 941d224efa9f..9d65c53d1789 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java @@ -50,6 +50,8 @@ void testRequestAndResponseWireFormat() throws Exception { assertThat((String) requestMap.get("idempotencyToken")).isNotBlank(); assertThat(RESTApi.fromJson(RESTApi.toJson(request), Map.class).get("idempotencyToken")) .isEqualTo(requestMap.get("idempotencyToken")); + assertThat(request.toPredictionSpec().idempotencyToken()) + .isEqualTo(request.getIdempotencyToken()); assertThat(requestMap) .containsEntry("instructions", "Classify") .doesNotContainKeys( diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 69ace5880b3d..9df117df2609 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -1226,7 +1226,10 @@ public Prediction submitPrediction(Identifier identifier, PredictionSpec spec) if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_MODEL)) { throw new ModelNotExistException(spec.model(), e); } - throw new TableNotExistException(identifier, e); + if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_TABLE)) { + throw new TableNotExistException(identifier, e); + } + throw e; } catch (NotImplementedException e) { throw new UnsupportedOperationException("REST Catalog does not support predictions", e); } @@ -1237,7 +1240,10 @@ public Prediction getPrediction(String predictionId) throws PredictionNotExistEx try { return api.getPrediction(predictionId).toPrediction(); } catch (NoSuchResourceException e) { - throw new PredictionNotExistException(predictionId, e); + if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_PREDICTION)) { + throw new PredictionNotExistException(predictionId, e); + } + throw e; } catch (NotImplementedException e) { throw new UnsupportedOperationException("REST Catalog does not support predictions", e); } @@ -1270,7 +1276,10 @@ public Prediction cancelPrediction(String predictionId) throws PredictionNotExis try { return api.cancelPrediction(predictionId).toPrediction(); } catch (NoSuchResourceException e) { - throw new PredictionNotExistException(predictionId, e); + if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_PREDICTION)) { + throw new PredictionNotExistException(predictionId, e); + } + throw e; } catch (NotImplementedException e) { throw new UnsupportedOperationException("REST Catalog does not support predictions", e); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java index 5ceb14856343..311adecf8ce7 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java @@ -26,6 +26,7 @@ import org.apache.paimon.catalog.PredictionSpec; import org.apache.paimon.catalog.PredictionStatus; import org.apache.paimon.options.Options; +import org.apache.paimon.rest.exceptions.NoSuchResourceException; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; @@ -106,9 +107,25 @@ void testPredictionNotFoundPreservesResourceType() throws Exception { assertThatThrownBy(() -> catalog.submitPrediction(table, spec())) .isInstanceOf(Catalog.ModelNotExistException.class); + server.enqueue(notFound("TABLE", "clips")); + assertThatThrownBy(() -> catalog.submitPrediction(table, spec())) + .isInstanceOf(Catalog.TableNotExistException.class); + + server.enqueue(untypedNotFound()); + assertThatThrownBy(() -> catalog.submitPrediction(table, spec())) + .isInstanceOf(NoSuchResourceException.class); + server.enqueue(notFound("PREDICTION", "prediction-1")); assertThatThrownBy(() -> catalog.getPrediction("prediction-1")) .isInstanceOf(Catalog.PredictionNotExistException.class); + + server.enqueue(untypedNotFound()); + assertThatThrownBy(() -> catalog.getPrediction("prediction-1")) + .isInstanceOf(NoSuchResourceException.class); + + server.enqueue(untypedNotFound()); + assertThatThrownBy(() -> catalog.cancelPrediction("prediction-1")) + .isInstanceOf(NoSuchResourceException.class); } } @@ -191,4 +208,8 @@ private static MockResponse notFound(String resourceType, String resourceName) { + "\",\"message\":\"not found\",\"code\":404}", 404); } + + private static MockResponse untypedNotFound() { + return json("{\"message\":\"route not found\",\"code\":404}", 404); + } } diff --git a/paimon-python/pypaimon/catalog/rest/rest_catalog.py b/paimon-python/pypaimon/catalog/rest/rest_catalog.py index 187cbd962181..52ade9da4583 100644 --- a/paimon-python/pypaimon/catalog/rest/rest_catalog.py +++ b/paimon-python/pypaimon/catalog/rest/rest_catalog.py @@ -696,7 +696,9 @@ def submit_prediction( except NoSuchResourceException as e: if e.resource_type == ErrorResponse.RESOURCE_TYPE_MODEL: raise ModelNotExistException(spec.model) from e - raise TableNotExistException(identifier) from e + if e.resource_type == ErrorResponse.RESOURCE_TYPE_TABLE: + raise TableNotExistException(identifier) from e + raise except NotImplementedException as e: raise NotImplementedError( "REST Catalog does not support predictions") from e @@ -708,7 +710,9 @@ def get_prediction( try: return self.rest_api.get_prediction(prediction_id).to_prediction() except NoSuchResourceException as e: - raise PredictionNotExistException(prediction_id) from e + if e.resource_type == ErrorResponse.RESOURCE_TYPE_PREDICTION: + raise PredictionNotExistException(prediction_id) from e + raise except NotImplementedException as e: raise NotImplementedError( "REST Catalog does not support predictions") from e @@ -741,7 +745,9 @@ def cancel_prediction( try: return self.rest_api.cancel_prediction(prediction_id).to_prediction() except NoSuchResourceException as e: - raise PredictionNotExistException(prediction_id) from e + if e.resource_type == ErrorResponse.RESOURCE_TYPE_PREDICTION: + raise PredictionNotExistException(prediction_id) from e + raise except NotImplementedException as e: raise NotImplementedError( "REST Catalog does not support predictions") from e diff --git a/paimon-python/pypaimon/common/wait.py b/paimon-python/pypaimon/common/wait.py index a2838ee1cf2b..99b93f51a275 100644 --- a/paimon-python/pypaimon/common/wait.py +++ b/paimon-python/pypaimon/common/wait.py @@ -30,7 +30,10 @@ def __init__(self, waiter: Callable, response=None, **kwargs): self._bind = kwargs def __getattr__(self, name): - return self._bind[name] + try: + return self._bind[name] + except KeyError: + raise AttributeError(name) from None def bind(self): return dict(self._bind) diff --git a/paimon-python/pypaimon/multimodal/prediction_wait.py b/paimon-python/pypaimon/multimodal/prediction_wait.py index 111aab6b97c0..5f0d400edc92 100644 --- a/paimon-python/pypaimon/multimodal/prediction_wait.py +++ b/paimon-python/pypaimon/multimodal/prediction_wait.py @@ -51,6 +51,8 @@ def _wait_for_prediction( attempt = 1 while time.monotonic() < deadline: prediction = catalog.get_prediction(prediction_id) + if time.monotonic() >= deadline: + break if prediction.status.terminal: return prediction if callback is not None: diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index 7ab325611c85..099cf90fd031 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -23,6 +23,7 @@ from unittest.mock import Mock, patch from pypaimon.api.api_response import PagedList +from pypaimon.api.rest_exception import NoSuchResourceException from pypaimon.api.prediction import ( GetPredictionResponse, ListPredictionsResponse, @@ -362,6 +363,49 @@ def test_result_timeout_does_not_cancel_prediction(self, monotonic): catalog.get_prediction.assert_not_called() catalog.cancel_prediction.assert_not_called() + @patch( + "pypaimon.multimodal.prediction_wait.time.monotonic", + side_effect=[0, 0, 0.2], + ) + def test_result_rejects_terminal_response_after_deadline(self, monotonic): + catalog = Mock() + catalog.get_prediction.return_value = _prediction( + PredictionStatus( + PredictionState.TERMINATED, + PredictionTerminationDetails( + PredictionTerminationCode.SUCCESS), + )) + waiter = PredictionWait(catalog, _prediction()) + + with self.assertRaisesRegex( + TimeoutError, "Prediction prediction-1 did not finish"): + waiter.result(timeout=timedelta(milliseconds=10)) + + catalog.get_prediction.assert_called_once_with("prediction-1") + + def test_wait_missing_attribute_uses_attribute_error(self): + waiter = PredictionWait(Mock(), _prediction()) + with self.assertRaises(AttributeError): + waiter.unknown_attribute + + def test_untyped_prediction_404_is_not_mapped_to_missing_resource(self): + catalog = object.__new__(RESTCatalog) + catalog.rest_api = Mock() + untyped = NoSuchResourceException(None, None, "route not found") + + catalog.rest_api.submit_prediction.side_effect = untyped + with self.assertRaises(NoSuchResourceException): + catalog.submit_prediction( + Identifier.create("quality", "clips"), _spec()) + + catalog.rest_api.get_prediction.side_effect = untyped + with self.assertRaises(NoSuchResourceException): + catalog.get_prediction("prediction-1") + + catalog.rest_api.cancel_prediction.side_effect = untyped + with self.assertRaises(NoSuchResourceException): + catalog.cancel_prediction("prediction-1") + def test_rest_round_trip_paths_and_filters(self): calls = [] From 4b62afa141d4ff653263a59863914cfd4cea2948 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Wed, 18 Nov 2026 15:50:35 +0800 Subject: [PATCH 29/41] Enforce prediction polling timeout budget --- paimon-python/pypaimon/api/client.py | 49 +++++++++++++++++-- paimon-python/pypaimon/api/rest_api.py | 7 ++- paimon-python/pypaimon/catalog/catalog.py | 3 +- .../pypaimon/catalog/rest/rest_catalog.py | 6 ++- .../pypaimon/multimodal/prediction_wait.py | 13 ++++- .../pypaimon/tests/rest/client_test.py | 33 ++++++++++++- .../pypaimon/tests/rest/prediction_test.py | 12 ++++- 7 files changed, 110 insertions(+), 13 deletions(-) diff --git a/paimon-python/pypaimon/api/client.py b/paimon-python/pypaimon/api/client.py index d2c179e718e5..6a5af5ca6810 100644 --- a/paimon-python/pypaimon/api/client.py +++ b/paimon-python/pypaimon/api/client.py @@ -25,6 +25,7 @@ import requests from requests.adapters import HTTPAdapter from urllib3 import Retry +from urllib3.exceptions import TimeoutError as Urllib3TimeoutError from pypaimon.api.api_response import ErrorResponse from pypaimon.api.rest_exception import (AlreadyExistsException, @@ -43,6 +44,33 @@ T = TypeVar('T', bound='RESTResponse') +def _is_transport_timeout(error: BaseException) -> bool: + pending = [error] + seen = set() + while pending: + current = pending.pop() + if id(current) in seen: + continue + seen.add(id(current)) + if isinstance(current, ( + requests.exceptions.Timeout, + Urllib3TimeoutError, + TimeoutError, + )): + return True + for nested in ( + getattr(current, "reason", None), + getattr(current, "__cause__", None), + getattr(current, "__context__", None), + ): + if isinstance(nested, BaseException): + pending.append(nested) + pending.extend( + value for value in getattr(current, "args", ()) + if isinstance(value, BaseException)) + return False + + class RESTRequest(ABC): """RESTRequest""" @@ -188,7 +216,8 @@ class RESTClient(ABC): @abstractmethod def get(self, path: str, response_type: Type[T], - rest_auth_function: Callable[[RESTAuthParameter], Dict[str, str]]) -> T: + rest_auth_function: Callable[[RESTAuthParameter], Dict[str, str]], + timeout_seconds: Optional[float] = None) -> T: pass @abstractmethod @@ -315,12 +344,14 @@ def set_error_handler(self, error_handler: ErrorHandler) -> None: self.error_handler = error_handler def get(self, path: str, response_type: Type[T], - rest_auth_function: Callable[[RESTAuthParameter], Dict[str, str]]) -> T: + rest_auth_function: Callable[[RESTAuthParameter], Dict[str, str]], + timeout_seconds: Optional[float] = None) -> T: auth_headers = _get_headers(path, "GET", {}, "", rest_auth_function) url = self._get_request_url(path, None) return self._execute_request("GET", url, headers=auth_headers, - response_type=response_type) + response_type=response_type, + timeout_seconds=timeout_seconds) def get_with_params(self, path: str, query_params: Dict[str, str], response_type: Type[T], @@ -386,7 +417,8 @@ def get_uri(self) -> str: def _execute_request(self, method: str, url: str, data: Optional[str] = None, headers: Optional[Dict[str, str]] = None, - response_type: Optional[Type[T]] = None) -> T: + response_type: Optional[Type[T]] = None, + timeout_seconds: Optional[float] = None) -> T: try: start_time = int(time.time() * 1_000_000_000) response = self.session.request( @@ -394,7 +426,8 @@ def _execute_request(self, method: str, url: str, url=url, data=data.encode('utf-8') if data else None, headers=headers, - timeout=self._timeout, + timeout=(self._timeout if timeout_seconds is None + else timeout_seconds), ) duration_ms = (int(time.time() * 1_000_000_000) - start_time) // 1_000_000 response_request_id = response.headers.get(self.REQUEST_ID_KEY, self.DEFAULT_REQUEST_ID) @@ -423,5 +456,11 @@ def _execute_request(self, method: str, url: str, except RESTException as e: raise e + except requests.exceptions.RequestException as e: + if (timeout_seconds is not None + and _is_transport_timeout(e)): + raise TimeoutError( + "REST request exceeded its time budget") from e + raise RESTException("rest exception", cause=e) except Exception as e: raise RESTException("rest exception", cause=e) diff --git a/paimon-python/pypaimon/api/rest_api.py b/paimon-python/pypaimon/api/rest_api.py index 84d4910aeb23..aa7880ae6f65 100755 --- a/paimon-python/pypaimon/api/rest_api.py +++ b/paimon-python/pypaimon/api/rest_api.py @@ -801,11 +801,16 @@ def submit_prediction( self.rest_auth_function, ) - def get_prediction(self, prediction_id: str) -> GetPredictionResponse: + def get_prediction( + self, + prediction_id: str, + timeout_seconds: Optional[float] = None, + ) -> GetPredictionResponse: return self.client.get( self.resource_paths.prediction(prediction_id), GetPredictionResponse, self.rest_auth_function, + timeout_seconds=timeout_seconds, ) def list_predictions( diff --git a/paimon-python/pypaimon/catalog/catalog.py b/paimon-python/pypaimon/catalog/catalog.py index 0fab39ac6c4c..72243a2c5c64 100644 --- a/paimon-python/pypaimon/catalog/catalog.py +++ b/paimon-python/pypaimon/catalog/catalog.py @@ -123,8 +123,9 @@ def submit_prediction( def get_prediction( self, prediction_id: str, + timeout_seconds: Optional[float] = None, ) -> 'Prediction': - """Get one prediction by its unique id.""" + """Get one prediction, optionally bounding the REST request in seconds.""" raise NotImplementedError( "get_prediction is not supported by this catalog.") diff --git a/paimon-python/pypaimon/catalog/rest/rest_catalog.py b/paimon-python/pypaimon/catalog/rest/rest_catalog.py index 52ade9da4583..f0e212666ffb 100644 --- a/paimon-python/pypaimon/catalog/rest/rest_catalog.py +++ b/paimon-python/pypaimon/catalog/rest/rest_catalog.py @@ -706,9 +706,13 @@ def submit_prediction( def get_prediction( self, prediction_id: str, + timeout_seconds: Optional[float] = None, ) -> Prediction: try: - return self.rest_api.get_prediction(prediction_id).to_prediction() + return self.rest_api.get_prediction( + prediction_id, + timeout_seconds=timeout_seconds, + ).to_prediction() except NoSuchResourceException as e: if e.resource_type == ErrorResponse.RESOURCE_TYPE_PREDICTION: raise PredictionNotExistException(prediction_id) from e diff --git a/paimon-python/pypaimon/multimodal/prediction_wait.py b/paimon-python/pypaimon/multimodal/prediction_wait.py index 5f0d400edc92..b738450c5822 100644 --- a/paimon-python/pypaimon/multimodal/prediction_wait.py +++ b/paimon-python/pypaimon/multimodal/prediction_wait.py @@ -49,8 +49,17 @@ def _wait_for_prediction( deadline = time.monotonic() + timeout.total_seconds() attempt = 1 - while time.monotonic() < deadline: - prediction = catalog.get_prediction(prediction_id) + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + break + try: + prediction = catalog.get_prediction( + prediction_id, timeout_seconds=remaining) + except TimeoutError as e: + raise TimeoutError( + "Prediction {} did not finish within {}".format( + prediction_id, timeout)) from e if time.monotonic() >= deadline: break if prediction.status.terminal: diff --git a/paimon-python/pypaimon/tests/rest/client_test.py b/paimon-python/pypaimon/tests/rest/client_test.py index f064cca8c3c7..d2f7f1abd048 100644 --- a/paimon-python/pypaimon/tests/rest/client_test.py +++ b/paimon-python/pypaimon/tests/rest/client_test.py @@ -18,6 +18,9 @@ import unittest from unittest import mock +import requests +from urllib3.exceptions import MaxRetryError, ReadTimeoutError + from pypaimon.api.client import HttpClient, _parse_error_response from pypaimon.api.rest_exception import (AlreadyExistsException, NoSuchResourceException, ServiceUnavailableException) @@ -133,11 +136,39 @@ def test_session_request_receives_timeout_tuple(self): client._execute_request('GET', 'http://localhost:8080/x') self.assertTrue(req.called, "session.request must be invoked") - kwargs = req.call_args.kwargs + kwargs = req.call_args[1] self.assertEqual( kwargs.get('timeout'), client._timeout, "request must carry the timeout; Session.timeout is dead code") + def test_explicit_request_timeout_overrides_default(self): + client = HttpClient("http://localhost:8080") + with mock.patch.object(client.session, 'request') as req: + req.return_value = mock.Mock( + status_code=200, text='{}', headers={}, + request=mock.Mock(method='GET'), + url='http://localhost:8080/x', + ok=True, + ) + client._execute_request( + 'GET', 'http://localhost:8080/x', timeout_seconds=0.01) + + self.assertEqual(0.01, req.call_args[1].get('timeout')) + + def test_explicit_timeout_unwraps_retry_error(self): + client = HttpClient("http://localhost:8080") + read_timeout = ReadTimeoutError( + None, "/x", "read timed out") + wrapped = requests.exceptions.ConnectionError( + MaxRetryError(None, "/x", read_timeout)) + + with mock.patch.object( + client.session, 'request', side_effect=wrapped): + with self.assertRaises(TimeoutError): + client._execute_request( + 'GET', 'http://localhost:8080/x', + timeout_seconds=0.01) + if __name__ == '__main__': unittest.main() diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index 099cf90fd031..6470d5e7c3a3 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -381,7 +381,10 @@ def test_result_rejects_terminal_response_after_deadline(self, monotonic): TimeoutError, "Prediction prediction-1 did not finish"): waiter.result(timeout=timedelta(milliseconds=10)) - catalog.get_prediction.assert_called_once_with("prediction-1") + catalog.get_prediction.assert_called_once() + call = catalog.get_prediction.call_args + self.assertEqual("prediction-1", call[0][0]) + self.assertAlmostEqual(0.01, call[1]["timeout_seconds"]) def test_wait_missing_attribute_uses_attribute_error(self): waiter = PredictionWait(Mock(), _prediction()) @@ -480,7 +483,12 @@ def do_GET(self): self.assertIn("maxResults=10", calls[-1][1]) self.assertIn("pageToken=p1", calls[-1][1]) - catalog.get_prediction("prediction/1") + with patch.object( + api.client, "get", wraps=api.client.get) as get: + catalog.get_prediction( + "prediction/1", timeout_seconds=0.25) + self.assertEqual( + 0.25, get.call_args[1]["timeout_seconds"]) self.assertTrue( calls[-1][1].endswith("/predictions/prediction%2F1")) catalog.cancel_prediction("prediction/1") From fe51bad9a9819aecf33b1314c3083838d0829f08 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Wed, 18 Nov 2026 16:03:43 +0800 Subject: [PATCH 30/41] Enforce prediction wait deadline --- .../pypaimon/multimodal/prediction_wait.py | 42 +++++- .../pypaimon/tests/rest/prediction_test.py | 124 ++++++++++++++++-- 2 files changed, 152 insertions(+), 14 deletions(-) diff --git a/paimon-python/pypaimon/multimodal/prediction_wait.py b/paimon-python/pypaimon/multimodal/prediction_wait.py index b738450c5822..5522da96f4dc 100644 --- a/paimon-python/pypaimon/multimodal/prediction_wait.py +++ b/paimon-python/pypaimon/multimodal/prediction_wait.py @@ -18,6 +18,8 @@ import random import time from datetime import timedelta +from queue import Empty, Queue +from threading import Thread from typing import Callable, Optional from pypaimon.catalog.prediction import Prediction @@ -54,8 +56,8 @@ def _wait_for_prediction( if remaining <= 0: break try: - prediction = catalog.get_prediction( - prediction_id, timeout_seconds=remaining) + prediction = _get_prediction_with_deadline( + catalog, prediction_id, deadline) except TimeoutError as e: raise TimeoutError( "Prediction {} did not finish within {}".format( @@ -76,3 +78,39 @@ def _wait_for_prediction( raise TimeoutError( "Prediction {} did not finish within {}".format( prediction_id, timeout)) + + +def _get_prediction_with_deadline( + catalog, + prediction_id: str, + deadline: float, +) -> Prediction: + completed = Queue(maxsize=1) + + def get_prediction(): + try: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError( + "Prediction GET exceeded its time budget") + completed.put(( + True, + catalog.get_prediction( + prediction_id, timeout_seconds=remaining), + )) + except Exception as error: + completed.put((False, error)) + + worker = Thread(target=get_prediction) + worker.daemon = True + worker.start() + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("Prediction GET exceeded its time budget") + try: + succeeded, value = completed.get(timeout=remaining) + except Empty: + raise TimeoutError("Prediction GET exceeded its time budget") + if succeeded: + return value + raise value diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index 6470d5e7c3a3..28d912a0152c 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -17,6 +17,7 @@ import json import threading +import time import unittest from datetime import timedelta from http.server import BaseHTTPRequestHandler, HTTPServer @@ -363,18 +364,21 @@ def test_result_timeout_does_not_cancel_prediction(self, monotonic): catalog.get_prediction.assert_not_called() catalog.cancel_prediction.assert_not_called() - @patch( - "pypaimon.multimodal.prediction_wait.time.monotonic", - side_effect=[0, 0, 0.2], - ) - def test_result_rejects_terminal_response_after_deadline(self, monotonic): + def test_result_rejects_terminal_response_after_deadline(self): + completed = threading.Event() catalog = Mock() - catalog.get_prediction.return_value = _prediction( - PredictionStatus( - PredictionState.TERMINATED, - PredictionTerminationDetails( - PredictionTerminationCode.SUCCESS), - )) + + def slow_terminal(*args, **kwargs): + time.sleep(0.05) + completed.set() + return _prediction( + PredictionStatus( + PredictionState.TERMINATED, + PredictionTerminationDetails( + PredictionTerminationCode.SUCCESS), + )) + + catalog.get_prediction.side_effect = slow_terminal waiter = PredictionWait(catalog, _prediction()) with self.assertRaisesRegex( @@ -384,7 +388,85 @@ def test_result_rejects_terminal_response_after_deadline(self, monotonic): catalog.get_prediction.assert_called_once() call = catalog.get_prediction.call_args self.assertEqual("prediction-1", call[0][0]) - self.assertAlmostEqual(0.01, call[1]["timeout_seconds"]) + self.assertGreater(call[1]["timeout_seconds"], 0) + self.assertLessEqual(call[1]["timeout_seconds"], 0.01) + self.assertTrue(completed.wait(timeout=1)) + + def test_result_timeout_covers_retry_after(self): + completed = threading.Event() + requests = [] + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_GET(self): + requests.append(self.path) + if len(requests) == 1: + body = b'{"message":"busy","code":503}' + self.send_response(503) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("Retry-After", "1") + self.end_headers() + self.wfile.write(body) + return + self._send_prediction() + completed.set() + + def _send_prediction(self): + body = JSON.to_json(_response()).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + server, thread, catalog = self._prediction_server(Handler) + try: + waiter = PredictionWait(catalog, _prediction()) + started = time.monotonic() + with self.assertRaises(TimeoutError): + waiter.result(timeout=timedelta(milliseconds=20)) + self.assertLess(time.monotonic() - started, 0.5) + self.assertTrue(completed.wait(timeout=2)) + finally: + server.shutdown() + server.server_close() + thread.join() + + def test_result_timeout_covers_slow_response_body(self): + completed = threading.Event() + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_GET(self): + body = JSON.to_json(_response()).encode() + chunk_size = max(1, (len(body) + 19) // 20) + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + for offset in range(0, len(body), chunk_size): + time.sleep(0.04) + self.wfile.write(body[offset:offset + chunk_size]) + self.wfile.flush() + completed.set() + + server, thread, catalog = self._prediction_server(Handler) + try: + waiter = PredictionWait(catalog, _prediction()) + started = time.monotonic() + with self.assertRaises(TimeoutError): + waiter.result(timeout=timedelta(milliseconds=100)) + self.assertLess(time.monotonic() - started, 0.5) + self.assertTrue(completed.wait(timeout=2)) + finally: + server.shutdown() + server.server_close() + thread.join() def test_wait_missing_attribute_uses_attribute_error(self): waiter = PredictionWait(Mock(), _prediction()) @@ -409,6 +491,24 @@ def test_untyped_prediction_404_is_not_mapped_to_missing_resource(self): with self.assertRaises(NoSuchResourceException): catalog.cancel_prediction("prediction-1") + @staticmethod + def _prediction_server(handler): + server = HTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + api = RESTApi( + { + "uri": "http://127.0.0.1:%s" % server.server_port, + "token.provider": "bear", + "token": "test", + "prefix": "catalog", + }, + False, + ) + catalog = object.__new__(RESTCatalog) + catalog.rest_api = api + return server, thread, catalog + def test_rest_round_trip_paths_and_filters(self): calls = [] From fdc8b5360cb6aafbbb4db62bffc5e027106bc13a Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Wed, 18 Nov 2026 16:15:56 +0800 Subject: [PATCH 31/41] Simplify prediction wait polling --- paimon-python/pypaimon/api/client.py | 49 +------ paimon-python/pypaimon/api/rest_api.py | 7 +- paimon-python/pypaimon/catalog/catalog.py | 3 +- .../pypaimon/catalog/rest/rest_catalog.py | 6 +- paimon-python/pypaimon/common/wait.py | 3 +- .../pypaimon/multimodal/prediction_wait.py | 51 +------ .../pypaimon/tests/rest/client_test.py | 31 ---- .../pypaimon/tests/rest/prediction_test.py | 134 ++---------------- 8 files changed, 25 insertions(+), 259 deletions(-) diff --git a/paimon-python/pypaimon/api/client.py b/paimon-python/pypaimon/api/client.py index 6a5af5ca6810..d2c179e718e5 100644 --- a/paimon-python/pypaimon/api/client.py +++ b/paimon-python/pypaimon/api/client.py @@ -25,7 +25,6 @@ import requests from requests.adapters import HTTPAdapter from urllib3 import Retry -from urllib3.exceptions import TimeoutError as Urllib3TimeoutError from pypaimon.api.api_response import ErrorResponse from pypaimon.api.rest_exception import (AlreadyExistsException, @@ -44,33 +43,6 @@ T = TypeVar('T', bound='RESTResponse') -def _is_transport_timeout(error: BaseException) -> bool: - pending = [error] - seen = set() - while pending: - current = pending.pop() - if id(current) in seen: - continue - seen.add(id(current)) - if isinstance(current, ( - requests.exceptions.Timeout, - Urllib3TimeoutError, - TimeoutError, - )): - return True - for nested in ( - getattr(current, "reason", None), - getattr(current, "__cause__", None), - getattr(current, "__context__", None), - ): - if isinstance(nested, BaseException): - pending.append(nested) - pending.extend( - value for value in getattr(current, "args", ()) - if isinstance(value, BaseException)) - return False - - class RESTRequest(ABC): """RESTRequest""" @@ -216,8 +188,7 @@ class RESTClient(ABC): @abstractmethod def get(self, path: str, response_type: Type[T], - rest_auth_function: Callable[[RESTAuthParameter], Dict[str, str]], - timeout_seconds: Optional[float] = None) -> T: + rest_auth_function: Callable[[RESTAuthParameter], Dict[str, str]]) -> T: pass @abstractmethod @@ -344,14 +315,12 @@ def set_error_handler(self, error_handler: ErrorHandler) -> None: self.error_handler = error_handler def get(self, path: str, response_type: Type[T], - rest_auth_function: Callable[[RESTAuthParameter], Dict[str, str]], - timeout_seconds: Optional[float] = None) -> T: + rest_auth_function: Callable[[RESTAuthParameter], Dict[str, str]]) -> T: auth_headers = _get_headers(path, "GET", {}, "", rest_auth_function) url = self._get_request_url(path, None) return self._execute_request("GET", url, headers=auth_headers, - response_type=response_type, - timeout_seconds=timeout_seconds) + response_type=response_type) def get_with_params(self, path: str, query_params: Dict[str, str], response_type: Type[T], @@ -417,8 +386,7 @@ def get_uri(self) -> str: def _execute_request(self, method: str, url: str, data: Optional[str] = None, headers: Optional[Dict[str, str]] = None, - response_type: Optional[Type[T]] = None, - timeout_seconds: Optional[float] = None) -> T: + response_type: Optional[Type[T]] = None) -> T: try: start_time = int(time.time() * 1_000_000_000) response = self.session.request( @@ -426,8 +394,7 @@ def _execute_request(self, method: str, url: str, url=url, data=data.encode('utf-8') if data else None, headers=headers, - timeout=(self._timeout if timeout_seconds is None - else timeout_seconds), + timeout=self._timeout, ) duration_ms = (int(time.time() * 1_000_000_000) - start_time) // 1_000_000 response_request_id = response.headers.get(self.REQUEST_ID_KEY, self.DEFAULT_REQUEST_ID) @@ -456,11 +423,5 @@ def _execute_request(self, method: str, url: str, except RESTException as e: raise e - except requests.exceptions.RequestException as e: - if (timeout_seconds is not None - and _is_transport_timeout(e)): - raise TimeoutError( - "REST request exceeded its time budget") from e - raise RESTException("rest exception", cause=e) except Exception as e: raise RESTException("rest exception", cause=e) diff --git a/paimon-python/pypaimon/api/rest_api.py b/paimon-python/pypaimon/api/rest_api.py index aa7880ae6f65..84d4910aeb23 100755 --- a/paimon-python/pypaimon/api/rest_api.py +++ b/paimon-python/pypaimon/api/rest_api.py @@ -801,16 +801,11 @@ def submit_prediction( self.rest_auth_function, ) - def get_prediction( - self, - prediction_id: str, - timeout_seconds: Optional[float] = None, - ) -> GetPredictionResponse: + def get_prediction(self, prediction_id: str) -> GetPredictionResponse: return self.client.get( self.resource_paths.prediction(prediction_id), GetPredictionResponse, self.rest_auth_function, - timeout_seconds=timeout_seconds, ) def list_predictions( diff --git a/paimon-python/pypaimon/catalog/catalog.py b/paimon-python/pypaimon/catalog/catalog.py index 72243a2c5c64..0fab39ac6c4c 100644 --- a/paimon-python/pypaimon/catalog/catalog.py +++ b/paimon-python/pypaimon/catalog/catalog.py @@ -123,9 +123,8 @@ def submit_prediction( def get_prediction( self, prediction_id: str, - timeout_seconds: Optional[float] = None, ) -> 'Prediction': - """Get one prediction, optionally bounding the REST request in seconds.""" + """Get one prediction by its unique id.""" raise NotImplementedError( "get_prediction is not supported by this catalog.") diff --git a/paimon-python/pypaimon/catalog/rest/rest_catalog.py b/paimon-python/pypaimon/catalog/rest/rest_catalog.py index f0e212666ffb..52ade9da4583 100644 --- a/paimon-python/pypaimon/catalog/rest/rest_catalog.py +++ b/paimon-python/pypaimon/catalog/rest/rest_catalog.py @@ -706,13 +706,9 @@ def submit_prediction( def get_prediction( self, prediction_id: str, - timeout_seconds: Optional[float] = None, ) -> Prediction: try: - return self.rest_api.get_prediction( - prediction_id, - timeout_seconds=timeout_seconds, - ).to_prediction() + return self.rest_api.get_prediction(prediction_id).to_prediction() except NoSuchResourceException as e: if e.resource_type == ErrorResponse.RESOURCE_TYPE_PREDICTION: raise PredictionNotExistException(prediction_id) from e diff --git a/paimon-python/pypaimon/common/wait.py b/paimon-python/pypaimon/common/wait.py index 99b93f51a275..c23cefdd4d19 100644 --- a/paimon-python/pypaimon/common/wait.py +++ b/paimon-python/pypaimon/common/wait.py @@ -45,7 +45,8 @@ def result( ) -> T: """Delegate waiting and return the final value. - A timeout only stops this client-side wait; it does not cancel the + The timeout is checked between polling requests. An in-flight request + may finish after the deadline, and timing out does not cancel the server-side operation. """ kwargs = self._bind.copy() diff --git a/paimon-python/pypaimon/multimodal/prediction_wait.py b/paimon-python/pypaimon/multimodal/prediction_wait.py index 5522da96f4dc..5f0d400edc92 100644 --- a/paimon-python/pypaimon/multimodal/prediction_wait.py +++ b/paimon-python/pypaimon/multimodal/prediction_wait.py @@ -18,8 +18,6 @@ import random import time from datetime import timedelta -from queue import Empty, Queue -from threading import Thread from typing import Callable, Optional from pypaimon.catalog.prediction import Prediction @@ -51,17 +49,8 @@ def _wait_for_prediction( deadline = time.monotonic() + timeout.total_seconds() attempt = 1 - while True: - remaining = deadline - time.monotonic() - if remaining <= 0: - break - try: - prediction = _get_prediction_with_deadline( - catalog, prediction_id, deadline) - except TimeoutError as e: - raise TimeoutError( - "Prediction {} did not finish within {}".format( - prediction_id, timeout)) from e + while time.monotonic() < deadline: + prediction = catalog.get_prediction(prediction_id) if time.monotonic() >= deadline: break if prediction.status.terminal: @@ -78,39 +67,3 @@ def _wait_for_prediction( raise TimeoutError( "Prediction {} did not finish within {}".format( prediction_id, timeout)) - - -def _get_prediction_with_deadline( - catalog, - prediction_id: str, - deadline: float, -) -> Prediction: - completed = Queue(maxsize=1) - - def get_prediction(): - try: - remaining = deadline - time.monotonic() - if remaining <= 0: - raise TimeoutError( - "Prediction GET exceeded its time budget") - completed.put(( - True, - catalog.get_prediction( - prediction_id, timeout_seconds=remaining), - )) - except Exception as error: - completed.put((False, error)) - - worker = Thread(target=get_prediction) - worker.daemon = True - worker.start() - remaining = deadline - time.monotonic() - if remaining <= 0: - raise TimeoutError("Prediction GET exceeded its time budget") - try: - succeeded, value = completed.get(timeout=remaining) - except Empty: - raise TimeoutError("Prediction GET exceeded its time budget") - if succeeded: - return value - raise value diff --git a/paimon-python/pypaimon/tests/rest/client_test.py b/paimon-python/pypaimon/tests/rest/client_test.py index d2f7f1abd048..4261c9ae4cf3 100644 --- a/paimon-python/pypaimon/tests/rest/client_test.py +++ b/paimon-python/pypaimon/tests/rest/client_test.py @@ -18,9 +18,6 @@ import unittest from unittest import mock -import requests -from urllib3.exceptions import MaxRetryError, ReadTimeoutError - from pypaimon.api.client import HttpClient, _parse_error_response from pypaimon.api.rest_exception import (AlreadyExistsException, NoSuchResourceException, ServiceUnavailableException) @@ -141,34 +138,6 @@ def test_session_request_receives_timeout_tuple(self): kwargs.get('timeout'), client._timeout, "request must carry the timeout; Session.timeout is dead code") - def test_explicit_request_timeout_overrides_default(self): - client = HttpClient("http://localhost:8080") - with mock.patch.object(client.session, 'request') as req: - req.return_value = mock.Mock( - status_code=200, text='{}', headers={}, - request=mock.Mock(method='GET'), - url='http://localhost:8080/x', - ok=True, - ) - client._execute_request( - 'GET', 'http://localhost:8080/x', timeout_seconds=0.01) - - self.assertEqual(0.01, req.call_args[1].get('timeout')) - - def test_explicit_timeout_unwraps_retry_error(self): - client = HttpClient("http://localhost:8080") - read_timeout = ReadTimeoutError( - None, "/x", "read timed out") - wrapped = requests.exceptions.ConnectionError( - MaxRetryError(None, "/x", read_timeout)) - - with mock.patch.object( - client.session, 'request', side_effect=wrapped): - with self.assertRaises(TimeoutError): - client._execute_request( - 'GET', 'http://localhost:8080/x', - timeout_seconds=0.01) - if __name__ == '__main__': unittest.main() diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index 28d912a0152c..099cf90fd031 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -17,7 +17,6 @@ import json import threading -import time import unittest from datetime import timedelta from http.server import BaseHTTPRequestHandler, HTTPServer @@ -364,109 +363,25 @@ def test_result_timeout_does_not_cancel_prediction(self, monotonic): catalog.get_prediction.assert_not_called() catalog.cancel_prediction.assert_not_called() - def test_result_rejects_terminal_response_after_deadline(self): - completed = threading.Event() + @patch( + "pypaimon.multimodal.prediction_wait.time.monotonic", + side_effect=[0, 0, 0.2], + ) + def test_result_rejects_terminal_response_after_deadline(self, monotonic): catalog = Mock() - - def slow_terminal(*args, **kwargs): - time.sleep(0.05) - completed.set() - return _prediction( - PredictionStatus( - PredictionState.TERMINATED, - PredictionTerminationDetails( - PredictionTerminationCode.SUCCESS), - )) - - catalog.get_prediction.side_effect = slow_terminal + catalog.get_prediction.return_value = _prediction( + PredictionStatus( + PredictionState.TERMINATED, + PredictionTerminationDetails( + PredictionTerminationCode.SUCCESS), + )) waiter = PredictionWait(catalog, _prediction()) with self.assertRaisesRegex( TimeoutError, "Prediction prediction-1 did not finish"): waiter.result(timeout=timedelta(milliseconds=10)) - catalog.get_prediction.assert_called_once() - call = catalog.get_prediction.call_args - self.assertEqual("prediction-1", call[0][0]) - self.assertGreater(call[1]["timeout_seconds"], 0) - self.assertLessEqual(call[1]["timeout_seconds"], 0.01) - self.assertTrue(completed.wait(timeout=1)) - - def test_result_timeout_covers_retry_after(self): - completed = threading.Event() - requests = [] - - class Handler(BaseHTTPRequestHandler): - def log_message(self, *args): - pass - - def do_GET(self): - requests.append(self.path) - if len(requests) == 1: - body = b'{"message":"busy","code":503}' - self.send_response(503) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.send_header("Retry-After", "1") - self.end_headers() - self.wfile.write(body) - return - self._send_prediction() - completed.set() - - def _send_prediction(self): - body = JSON.to_json(_response()).encode() - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - server, thread, catalog = self._prediction_server(Handler) - try: - waiter = PredictionWait(catalog, _prediction()) - started = time.monotonic() - with self.assertRaises(TimeoutError): - waiter.result(timeout=timedelta(milliseconds=20)) - self.assertLess(time.monotonic() - started, 0.5) - self.assertTrue(completed.wait(timeout=2)) - finally: - server.shutdown() - server.server_close() - thread.join() - - def test_result_timeout_covers_slow_response_body(self): - completed = threading.Event() - - class Handler(BaseHTTPRequestHandler): - def log_message(self, *args): - pass - - def do_GET(self): - body = JSON.to_json(_response()).encode() - chunk_size = max(1, (len(body) + 19) // 20) - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - for offset in range(0, len(body), chunk_size): - time.sleep(0.04) - self.wfile.write(body[offset:offset + chunk_size]) - self.wfile.flush() - completed.set() - - server, thread, catalog = self._prediction_server(Handler) - try: - waiter = PredictionWait(catalog, _prediction()) - started = time.monotonic() - with self.assertRaises(TimeoutError): - waiter.result(timeout=timedelta(milliseconds=100)) - self.assertLess(time.monotonic() - started, 0.5) - self.assertTrue(completed.wait(timeout=2)) - finally: - server.shutdown() - server.server_close() - thread.join() + catalog.get_prediction.assert_called_once_with("prediction-1") def test_wait_missing_attribute_uses_attribute_error(self): waiter = PredictionWait(Mock(), _prediction()) @@ -491,24 +406,6 @@ def test_untyped_prediction_404_is_not_mapped_to_missing_resource(self): with self.assertRaises(NoSuchResourceException): catalog.cancel_prediction("prediction-1") - @staticmethod - def _prediction_server(handler): - server = HTTPServer(("127.0.0.1", 0), handler) - thread = threading.Thread(target=server.serve_forever, daemon=True) - thread.start() - api = RESTApi( - { - "uri": "http://127.0.0.1:%s" % server.server_port, - "token.provider": "bear", - "token": "test", - "prefix": "catalog", - }, - False, - ) - catalog = object.__new__(RESTCatalog) - catalog.rest_api = api - return server, thread, catalog - def test_rest_round_trip_paths_and_filters(self): calls = [] @@ -583,12 +480,7 @@ def do_GET(self): self.assertIn("maxResults=10", calls[-1][1]) self.assertIn("pageToken=p1", calls[-1][1]) - with patch.object( - api.client, "get", wraps=api.client.get) as get: - catalog.get_prediction( - "prediction/1", timeout_seconds=0.25) - self.assertEqual( - 0.25, get.call_args[1]["timeout_seconds"]) + catalog.get_prediction("prediction/1") self.assertTrue( calls[-1][1].endswith("/predictions/prediction%2F1")) catalog.cancel_prediction("prediction/1") From 2da4c4ec27333db4dc96b8e4f09b2fdfe0701087 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Wed, 18 Nov 2026 16:54:00 +0800 Subject: [PATCH 32/41] Map missing prediction databases --- .../org/apache/paimon/catalog/Catalog.java | 2 +- .../apache/paimon/catalog/DelegateCatalog.java | 2 +- .../org/apache/paimon/rest/RESTCatalog.java | 5 ++++- .../paimon/rest/RESTCatalogPredictionTest.java | 4 ++++ .../pypaimon/catalog/rest/rest_catalog.py | 3 +++ .../pypaimon/tests/rest/prediction_test.py | 18 +++++++++++++++++- 6 files changed, 30 insertions(+), 4 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java index d3bac00bfb01..0eae9bb8f99a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java @@ -1201,7 +1201,7 @@ default void dropModel(Identifier identifier, boolean ignoreIfNotExists) /** Submit an asynchronous prediction over a table and its fixed snapshot. */ default Prediction submitPrediction(Identifier identifier, PredictionSpec spec) - throws TableNotExistException, ModelNotExistException { + throws DatabaseNotExistException, TableNotExistException, ModelNotExistException { throw new UnsupportedOperationException("Current Catalog does not support predictions"); } diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java index 60df74b60b9d..8b72c9496767 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java @@ -409,7 +409,7 @@ public void dropModel(Identifier identifier, boolean ignoreIfNotExists) @Override public Prediction submitPrediction(Identifier identifier, PredictionSpec spec) - throws TableNotExistException, ModelNotExistException { + throws DatabaseNotExistException, TableNotExistException, ModelNotExistException { return wrapped.submitPrediction(identifier, spec); } diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index 9df117df2609..c55778be2c2b 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -1216,13 +1216,16 @@ public void dropModel(Identifier identifier, boolean ignoreIfNotExists) @Override public Prediction submitPrediction(Identifier identifier, PredictionSpec spec) - throws TableNotExistException, ModelNotExistException { + throws DatabaseNotExistException, TableNotExistException, ModelNotExistException { checkNotBranch(identifier, "submitPrediction"); checkNotSystemTable(identifier, "submitPrediction"); try { return api.submitPrediction(new SubmitPredictionRequest(identifier, spec)) .toPrediction(); } catch (NoSuchResourceException e) { + if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_DATABASE)) { + throw new DatabaseNotExistException(identifier.getDatabaseName(), e); + } if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_MODEL)) { throw new ModelNotExistException(spec.model(), e); } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java index 311adecf8ce7..06f581ac4b58 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java @@ -103,6 +103,10 @@ void testPredictionNotFoundPreservesResourceType() throws Exception { Catalog catalog = catalog(server); Identifier table = Identifier.create("quality", "clips"); + server.enqueue(notFound("DATABASE", "quality")); + assertThatThrownBy(() -> catalog.submitPrediction(table, spec())) + .isInstanceOf(Catalog.DatabaseNotExistException.class); + server.enqueue(notFound("MODEL", "qwen_vl")); assertThatThrownBy(() -> catalog.submitPrediction(table, spec())) .isInstanceOf(Catalog.ModelNotExistException.class); diff --git a/paimon-python/pypaimon/catalog/rest/rest_catalog.py b/paimon-python/pypaimon/catalog/rest/rest_catalog.py index 52ade9da4583..b1e4119ddbda 100644 --- a/paimon-python/pypaimon/catalog/rest/rest_catalog.py +++ b/paimon-python/pypaimon/catalog/rest/rest_catalog.py @@ -694,6 +694,9 @@ def submit_prediction( return self.rest_api.submit_prediction( SubmitPredictionRequest.from_spec(identifier, spec)).to_prediction() except NoSuchResourceException as e: + if e.resource_type == ErrorResponse.RESOURCE_TYPE_DATABASE: + raise DatabaseNotExistException( + identifier.get_database_name()) from e if e.resource_type == ErrorResponse.RESOURCE_TYPE_MODEL: raise ModelNotExistException(spec.model) from e if e.resource_type == ErrorResponse.RESOURCE_TYPE_TABLE: diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index 099cf90fd031..f6a4f303d1fc 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -22,7 +22,7 @@ from http.server import BaseHTTPRequestHandler, HTTPServer from unittest.mock import Mock, patch -from pypaimon.api.api_response import PagedList +from pypaimon.api.api_response import ErrorResponse, PagedList from pypaimon.api.rest_exception import NoSuchResourceException from pypaimon.api.prediction import ( GetPredictionResponse, @@ -32,6 +32,7 @@ SubmitPredictionRequest, ) from pypaimon.api.rest_api import RESTApi +from pypaimon.catalog.catalog_exception import DatabaseNotExistException from pypaimon.catalog.prediction import ( Prediction, PredictionOutput, @@ -406,6 +407,21 @@ def test_untyped_prediction_404_is_not_mapped_to_missing_resource(self): with self.assertRaises(NoSuchResourceException): catalog.cancel_prediction("prediction-1") + def test_submit_prediction_maps_database_404(self): + catalog = object.__new__(RESTCatalog) + catalog.rest_api = Mock() + catalog.rest_api.submit_prediction.side_effect = NoSuchResourceException( + ErrorResponse.RESOURCE_TYPE_DATABASE, + "quality", + "database not found", + ) + + with self.assertRaises(DatabaseNotExistException) as context: + catalog.submit_prediction( + Identifier.create("quality", "clips"), _spec()) + + self.assertEqual("quality", context.exception.database) + def test_rest_round_trip_paths_and_filters(self): calls = [] From 5bbf41082d16b7db70be32bcaaa6672042802abd Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Wed, 18 Nov 2026 16:59:08 +0800 Subject: [PATCH 33/41] Report the missing prediction database --- .../src/main/java/org/apache/paimon/rest/RESTCatalog.java | 6 +++++- .../org/apache/paimon/rest/RESTCatalogPredictionTest.java | 5 +++-- paimon-python/pypaimon/catalog/rest/rest_catalog.py | 3 ++- paimon-python/pypaimon/tests/rest/prediction_test.py | 4 ++-- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index c55778be2c2b..d06405196110 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -1224,7 +1224,11 @@ public Prediction submitPrediction(Identifier identifier, PredictionSpec spec) .toPrediction(); } catch (NoSuchResourceException e) { if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_DATABASE)) { - throw new DatabaseNotExistException(identifier.getDatabaseName(), e); + String databaseName = + StringUtils.isNullOrWhitespaceOnly(e.resourceName()) + ? identifier.getDatabaseName() + : e.resourceName(); + throw new DatabaseNotExistException(databaseName, e); } if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_MODEL)) { throw new ModelNotExistException(spec.model(), e); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java index 06f581ac4b58..67b0f165cbc4 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java @@ -103,9 +103,10 @@ void testPredictionNotFoundPreservesResourceType() throws Exception { Catalog catalog = catalog(server); Identifier table = Identifier.create("quality", "clips"); - server.enqueue(notFound("DATABASE", "quality")); + server.enqueue(notFound("DATABASE", "models")); assertThatThrownBy(() -> catalog.submitPrediction(table, spec())) - .isInstanceOf(Catalog.DatabaseNotExistException.class); + .isInstanceOf(Catalog.DatabaseNotExistException.class) + .hasMessage("Database models does not exist."); server.enqueue(notFound("MODEL", "qwen_vl")); assertThatThrownBy(() -> catalog.submitPrediction(table, spec())) diff --git a/paimon-python/pypaimon/catalog/rest/rest_catalog.py b/paimon-python/pypaimon/catalog/rest/rest_catalog.py index b1e4119ddbda..2de8eca60d30 100644 --- a/paimon-python/pypaimon/catalog/rest/rest_catalog.py +++ b/paimon-python/pypaimon/catalog/rest/rest_catalog.py @@ -696,7 +696,8 @@ def submit_prediction( except NoSuchResourceException as e: if e.resource_type == ErrorResponse.RESOURCE_TYPE_DATABASE: raise DatabaseNotExistException( - identifier.get_database_name()) from e + e.resource_name + or identifier.get_database_name()) from e if e.resource_type == ErrorResponse.RESOURCE_TYPE_MODEL: raise ModelNotExistException(spec.model) from e if e.resource_type == ErrorResponse.RESOURCE_TYPE_TABLE: diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index f6a4f303d1fc..e1edce283218 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -412,7 +412,7 @@ def test_submit_prediction_maps_database_404(self): catalog.rest_api = Mock() catalog.rest_api.submit_prediction.side_effect = NoSuchResourceException( ErrorResponse.RESOURCE_TYPE_DATABASE, - "quality", + "models", "database not found", ) @@ -420,7 +420,7 @@ def test_submit_prediction_maps_database_404(self): catalog.submit_prediction( Identifier.create("quality", "clips"), _spec()) - self.assertEqual("quality", context.exception.database) + self.assertEqual("models", context.exception.database) def test_rest_round_trip_paths_and_filters(self): calls = [] From 8799d0f306e19bc2363ca4a1732888932b042319 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Wed, 18 Nov 2026 17:05:08 +0800 Subject: [PATCH 34/41] Reject prediction dot segments --- docs/static/rest-catalog-open-api.yaml | 12 +++++++++--- .../java/org/apache/paimon/rest/ResourcePaths.java | 10 ++++------ .../org/apache/paimon/rest/ResourcePathsTest.java | 9 +++++++++ paimon-python/pypaimon/api/resource_paths.py | 2 ++ .../pypaimon/tests/rest/resource_paths_test.py | 9 +++++++++ 5 files changed, 33 insertions(+), 9 deletions(-) diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 3c022a915415..6791eccc76e6 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -197,7 +197,7 @@ paths: operationId: getPrediction parameters: - {name: prefix, in: path, required: true, schema: {type: string}} - - {name: predictionId, in: path, required: true, schema: {type: string, minLength: 1, maxLength: 64}} + - {name: predictionId, in: path, required: true, schema: {$ref: '#/components/schemas/PredictionId'}} responses: "200": description: Prediction metadata @@ -218,7 +218,7 @@ paths: unchanged. parameters: - {name: prefix, in: path, required: true, schema: {type: string}} - - {name: predictionId, in: path, required: true, schema: {type: string, minLength: 1, maxLength: 64}} + - {name: predictionId, in: path, required: true, schema: {$ref: '#/components/schemas/PredictionId'}} responses: "200": description: Latest prediction metadata @@ -4589,6 +4589,12 @@ components: minLength: 1 pattern: '^(?!\.{1,2}$).+' description: A non-empty path segment other than `.` or `..`. + PredictionId: + type: string + minLength: 1 + maxLength: 64 + pattern: '^(?!\.{1,2}$).+' + description: A prediction identifier other than `.` or `..`. ModelField: type: object additionalProperties: false @@ -4791,7 +4797,7 @@ components: additionalProperties: false required: [predictionId, source, status, snapshotId, model] properties: - predictionId: {type: string, minLength: 1, maxLength: 64} + predictionId: {$ref: '#/components/schemas/PredictionId'} source: $ref: '#/components/schemas/PredictionSource' name: {type: string, minLength: 1, maxLength: 128} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java index 120d8eaafb91..8858524ca18b 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -152,12 +152,12 @@ public String database(String databaseName) { } public String models(String databaseName) { - checkModelPathSegment(databaseName, "databaseName"); + checkPathSegment(databaseName, "databaseName"); return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), MODELS); } public String model(String databaseName, String modelName) { - checkModelPathSegment(modelName, "modelName"); + checkPathSegment(modelName, "modelName"); return SLASH.join(models(databaseName), encodeString(modelName)); } @@ -166,9 +166,7 @@ public String predictions() { } public String prediction(String predictionId) { - checkArgument( - predictionId != null && !predictionId.trim().isEmpty(), - "predictionId must not be blank"); + checkPathSegment(predictionId, "predictionId"); return SLASH.join(predictions(), encodeString(predictionId)); } @@ -176,7 +174,7 @@ public String cancelPrediction(String predictionId) { return SLASH.join(prediction(predictionId), "cancel"); } - private static void checkModelPathSegment(String value, String name) { + private static void checkPathSegment(String value, String name) { checkArgument(value != null && !value.trim().isEmpty(), "%s must not be blank", name); checkArgument( !".".equals(value) && !"..".equals(value), "%s must not be a dot segment", name); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/ResourcePathsTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/ResourcePathsTest.java index b6f0f39f38f5..e31a2899a1b5 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/ResourcePathsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/ResourcePathsTest.java @@ -71,4 +71,13 @@ public void testPoliciesAreNestedUnderAttachmentResource() { "/v1/catalog%2Fid/databases/sales+db/tables/orders%2Fall/policies/drop", paths.dropPolicy(table)); } + + @Test + public void testPredictionRejectsDotSegments() { + ResourcePaths paths = new ResourcePaths("paimon"); + assertThrows(IllegalArgumentException.class, () -> paths.prediction(".")); + assertThrows(IllegalArgumentException.class, () -> paths.prediction("..")); + assertThrows(IllegalArgumentException.class, () -> paths.cancelPrediction(".")); + assertThrows(IllegalArgumentException.class, () -> paths.cancelPrediction("..")); + } } diff --git a/paimon-python/pypaimon/api/resource_paths.py b/paimon-python/pypaimon/api/resource_paths.py index 81cbc772d0c2..7fa70f76c007 100644 --- a/paimon-python/pypaimon/api/resource_paths.py +++ b/paimon-python/pypaimon/api/resource_paths.py @@ -139,6 +139,8 @@ def predictions(self) -> str: def prediction(self, prediction_id: str) -> str: if not isinstance(prediction_id, str) or not prediction_id.strip(): raise ValueError("prediction_id must be a non-empty string") + if prediction_id in (".", ".."): + raise ValueError("prediction_id must not be a dot segment") return "{}/{}".format( self.predictions(), RESTUtil.encode_string(prediction_id)) diff --git a/paimon-python/pypaimon/tests/rest/resource_paths_test.py b/paimon-python/pypaimon/tests/rest/resource_paths_test.py index eccd72f68f86..731e90901b89 100644 --- a/paimon-python/pypaimon/tests/rest/resource_paths_test.py +++ b/paimon-python/pypaimon/tests/rest/resource_paths_test.py @@ -51,6 +51,15 @@ def test_database_and_table_names_are_url_encoded(self): "/v1/catalog%2Fid/databases/sales+db/table-details", paths.table_details("sales db")) self.assertEqual("/v1/catalog%2Fid/tables/id/a%2Fb+c", paths.table_by_id("a/b c")) + def test_prediction_rejects_dot_segments(self): + paths = ResourcePaths("paimon") + for prediction_id in (".", ".."): + with self.subTest(prediction_id=prediction_id): + with self.assertRaises(ValueError): + paths.prediction(prediction_id) + with self.assertRaises(ValueError): + paths.cancel_prediction(prediction_id) + def test_signature_matches_the_path_sent(self): received = [] From 096958644f99c9f7cd90c51f68a18a8cb39280ed Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Thu, 19 Nov 2026 15:40:04 +0800 Subject: [PATCH 35/41] [core] Add executable prediction reference integration --- .../paimon/rest/MockRESTCatalogTest.java | 110 ++++ .../paimon/rest/RESTCatalogModelHandler.java | 7 + .../rest/RESTCatalogPredictionHandler.java | 621 ++++++++++++++++++ .../apache/paimon/rest/RESTCatalogServer.java | 19 + 4 files changed, 757 insertions(+) create mode 100644 paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java index ecf653f58c0f..ab9b518efaa8 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java @@ -27,8 +27,13 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.catalog.Model; import org.apache.paimon.catalog.ModelField; +import org.apache.paimon.catalog.Prediction; +import org.apache.paimon.catalog.PredictionSpec; +import org.apache.paimon.catalog.PredictionStatus; import org.apache.paimon.catalog.ProviderModelDefinition; import org.apache.paimon.catalog.TableQueryAuthResult; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericRow; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.management.ListPermissionsRequest; @@ -190,6 +195,111 @@ void testModelCrudRoundTrip() throws Exception { .isInstanceOf(Catalog.DatabaseNotExistException.class); } + @Test + void testPredictionReferenceIntegration() throws Exception { + String database = "prediction_reference"; + Identifier sourceIdentifier = Identifier.create(database, "clips"); + Identifier modelIdentifier = Identifier.create(database, "echo_model"); + catalog.createDatabase(database, false); + catalog.createTable( + sourceIdentifier, + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("content", DataTypes.STRING()) + .build(), + false); + FileStoreTable source = (FileStoreTable) catalog.getTable(sourceIdentifier); + commitRows(source, GenericRow.of(1, BinaryString.fromString("first"))); + + ProviderModelDefinition definition = + new ProviderModelDefinition( + "reference", + "echo", + Collections.singletonList(new ModelField("text", "STRING", null)), + Collections.singletonList(new ModelField("response", "STRING", null)), + Collections.emptyMap()); + catalog.createModel( + modelIdentifier, + new Model(modelIdentifier, "Deterministic reference model", definition), + false); + + Runnable resumePredictionExecution = restCatalogServer.pausePredictionExecution(); + Prediction submitted = + catalog.submitPrediction( + sourceIdentifier, + new PredictionSpec( + "snapshot-readback", + modelIdentifier, + Collections.singletonMap("text", "content"), + Collections.singletonList("id"), + "Echo the input", + null, + Collections.emptyMap(), + "reference-success")); + long pinnedSnapshot = submitted.snapshotId(); + + commitRows(source, GenericRow.of(2, BinaryString.fromString("second"))); + assertThat( + ((FileStoreTable) catalog.getTable(sourceIdentifier)) + .snapshotManager() + .latestSnapshotId()) + .isGreaterThan(pinnedSnapshot); + + resumePredictionExecution.run(); + Prediction completed = waitForPrediction(submitted.predictionId()); + assertThat(completed.status().terminationDetails().code()) + .isEqualTo(PredictionStatus.TerminationCode.SUCCESS); + assertThat(completed.snapshotId()).isEqualTo(pinnedSnapshot); + assertThat(completed.output()).isNotNull(); + assertThat(batchRead(catalog.getTable(completed.output().table().toIdentifier()))) + .containsExactly("+I[1, {\"text\":\"first\"}, null]"); + + resumePredictionExecution = restCatalogServer.pausePredictionExecution(); + Prediction toCancel = + catalog.submitPrediction( + sourceIdentifier, + new PredictionSpec( + "cancel-reference", + modelIdentifier, + Collections.singletonMap("text", "content"), + Collections.singletonList("id"), + "Echo the input", + null, + Collections.emptyMap(), + "reference-cancel")); + waitForPredictionState(toCancel.predictionId(), PredictionStatus.State.RUNNING); + Prediction canceled = catalog.cancelPrediction(toCancel.predictionId()); + assertThat(canceled.status().state()).isEqualTo(PredictionStatus.State.TERMINATED); + assertThat(canceled.status().terminationDetails().code()) + .isEqualTo(PredictionStatus.TerminationCode.CANCELED); + assertThat(canceled.output()).isNull(); + resumePredictionExecution.run(); + } + + private Prediction waitForPrediction(String predictionId) throws Exception { + long deadline = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(10); + while (System.currentTimeMillis() < deadline) { + Prediction prediction = catalog.getPrediction(predictionId); + if (prediction.status().isTerminal()) { + return prediction; + } + Thread.sleep(10L); + } + throw new AssertionError("Prediction did not terminate: " + predictionId); + } + + private void waitForPredictionState(String predictionId, PredictionStatus.State expectedState) + throws Exception { + long deadline = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(10); + while (System.currentTimeMillis() < deadline) { + if (catalog.getPrediction(predictionId).status().state() == expectedState) { + return; + } + Thread.sleep(10L); + } + throw new AssertionError("Prediction did not reach " + expectedState + ": " + predictionId); + } + @Test public void testAlterTableRethrowsForeignTypedNotFound() throws Exception { // a 404 typed other than TABLE/COLUMN must not be swallowed into reported success diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogModelHandler.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogModelHandler.java index 2ab0b27a7d82..afdd42f8455b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogModelHandler.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogModelHandler.java @@ -31,6 +31,8 @@ import okhttp3.mockwebserver.MockResponse; +import javax.annotation.Nullable; + import java.util.Comparator; import java.util.List; import java.util.Map; @@ -103,6 +105,11 @@ void removeDatabase(String databaseName) { .equals(databaseName)); } + @Nullable + Model getModel(Identifier identifier) { + return modelStore.get(identifier.getFullName()); + } + private MockResponse list(String databaseName, Map parameters) { int maxResults = maxResults(parameters); String pageToken = parameters.get(PAGE_TOKEN); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java new file mode 100644 index 000000000000..8e7a30018e3d --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java @@ -0,0 +1,621 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.Snapshot; +import org.apache.paimon.catalog.Database; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.Prediction; +import org.apache.paimon.catalog.PredictionOutput; +import org.apache.paimon.catalog.PredictionSource; +import org.apache.paimon.catalog.PredictionSpec; +import org.apache.paimon.catalog.PredictionStatus; +import org.apache.paimon.catalog.TableMetadata; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.reader.RecordReaderIterator; +import org.apache.paimon.rest.requests.SubmitPredictionRequest; +import org.apache.paimon.rest.responses.ErrorResponse; +import org.apache.paimon.rest.responses.GetPredictionResponse; +import org.apache.paimon.rest.responses.ListPredictionsResponse; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.sink.BatchTableCommit; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.utils.InternalRowUtils; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonProcessingException; + +import okhttp3.mockwebserver.MockResponse; + +import javax.annotation.Nullable; + +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; + +import static org.apache.paimon.rest.RESTApi.MAX_RESULTS; +import static org.apache.paimon.rest.RESTApi.PAGE_TOKEN; +import static org.apache.paimon.rest.RESTCatalogServerUtils.parseRequest; + +/** Executable prediction endpoints for the local REST catalog reference integration. */ +final class RESTCatalogPredictionHandler implements AutoCloseable { + + private static final String DATABASE = "database"; + private static final String TABLE = "table"; + private static final String NAME = "name"; + private static final String STATE = "state"; + + private final RESTCatalogServer server; + private final RESTFileSystemCatalog catalog; + private final RESTCatalogModelHandler modelHandler; + private final Map databaseStore; + private final Map tableMetadataStore; + private final String predictionsPath; + private final Map jobs = new ConcurrentHashMap<>(); + private final Map jobsByIdempotencyToken = new ConcurrentHashMap<>(); + private final AtomicLong nextId = new AtomicLong(); + private final ExecutorService executor; + + private volatile CountDownLatch executionGate = new CountDownLatch(0); + + RESTCatalogPredictionHandler( + RESTCatalogServer server, + RESTFileSystemCatalog catalog, + RESTCatalogModelHandler modelHandler, + Map databaseStore, + Map tableMetadataStore, + String predictionsPath) { + this.server = server; + this.catalog = catalog; + this.modelHandler = modelHandler; + this.databaseStore = databaseStore; + this.tableMetadataStore = tableMetadataStore; + this.predictionsPath = predictionsPath; + this.executor = + Executors.newSingleThreadExecutor( + runnable -> { + Thread thread = + new Thread(runnable, "paimon-reference-prediction-executor"); + thread.setDaemon(true); + return thread; + }); + } + + MockResponse handle( + String method, String resourcePath, String data, Map parameters) + throws Exception { + if (predictionsPath.equals(resourcePath)) { + if ("POST".equals(method)) { + return submit(data); + } else if ("GET".equals(method)) { + return list(parameters); + } + return new MockResponse().setResponseCode(404); + } + + String suffix = resourcePath.substring((predictionsPath + "/").length()); + String[] segments = suffix.split("/"); + String predictionId = RESTUtil.decodeString(segments[0]); + if (segments.length == 1 && "GET".equals(method)) { + return get(predictionId); + } + if (segments.length == 2 && "cancel".equals(segments[1]) && "POST".equals(method)) { + return cancel(predictionId); + } + return new MockResponse().setResponseCode(404); + } + + void pauseExecution() { + executionGate = new CountDownLatch(1); + } + + void resumeExecution() { + executionGate.countDown(); + } + + private MockResponse submit(String data) throws Exception { + SubmitPredictionRequest request = parseRequest(data, SubmitPredictionRequest.class); + PredictionJob existing = jobsByIdempotencyToken.get(request.getIdempotencyToken()); + if (existing != null) { + if (existing.requestPayload.equals(data)) { + return response(new GetPredictionResponse(existing.snapshot()), 202); + } + return error( + ErrorResponse.RESOURCE_TYPE_PREDICTION, + existing.predictionId, + "idempotencyToken is already used by another prediction", + 409); + } + + Identifier source = request.getSource().toIdentifier(); + PredictionSpec spec = request.toPredictionSpec(); + MockResponse validationError = validate(source, spec); + if (validationError != null) { + return validationError; + } + + FileStoreTable sourceTable = (FileStoreTable) catalog.getTable(source); + long snapshotId = resolveSnapshot(sourceTable, spec); + String predictionId = String.format("prediction-%020d", nextId.incrementAndGet()); + PredictionJob job = + new PredictionJob( + predictionId, request.getSource(), spec, snapshotId, data, executionGate); + jobs.put(predictionId, job); + jobsByIdempotencyToken.put(request.getIdempotencyToken(), job); + Future future = executor.submit(() -> execute(job)); + job.setFuture(future); + return response(new GetPredictionResponse(job.snapshot()), 202); + } + + @Nullable + private MockResponse validate(Identifier source, PredictionSpec spec) { + if (!databaseStore.containsKey(source.getDatabaseName())) { + return error( + ErrorResponse.RESOURCE_TYPE_DATABASE, + source.getDatabaseName(), + "Database does not exist", + 404); + } + if (!tableMetadataStore.containsKey(source.getFullName())) { + return error( + ErrorResponse.RESOURCE_TYPE_TABLE, + source.getObjectName(), + "Table does not exist", + 404); + } + Identifier model = spec.model(); + if (!databaseStore.containsKey(model.getDatabaseName())) { + return error( + ErrorResponse.RESOURCE_TYPE_DATABASE, + model.getDatabaseName(), + "Database does not exist", + 404); + } + if (modelHandler.getModel(model) == null) { + return error( + ErrorResponse.RESOURCE_TYPE_MODEL, + model.getObjectName(), + "Model does not exist", + 404); + } + if (spec.where() != null) { + return error(null, null, "The reference provider does not evaluate where", 501); + } + + TableSchema schema; + try { + schema = ((FileStoreTable) catalog.getTable(source)).schema(); + } catch (Exception e) { + return error( + ErrorResponse.RESOURCE_TYPE_TABLE, + source.getObjectName(), + "Table does not exist", + 404); + } + List fields = schema.fieldNames(); + for (String input : spec.inputs().values()) { + if (!fields.contains(input)) { + return error( + ErrorResponse.RESOURCE_TYPE_COLUMN, + input, + "Prediction input column does not exist", + 404); + } + } + for (String key : spec.mergeKeyColumns()) { + if (!fields.contains(key)) { + return error( + ErrorResponse.RESOURCE_TYPE_COLUMN, + key, + "Prediction merge key column does not exist", + 404); + } + if ("response".equals(key) || "error".equals(key)) { + return error(null, null, "mergeKeyColumns use a reserved output name", 400); + } + } + return null; + } + + private long resolveSnapshot(FileStoreTable table, PredictionSpec spec) { + CoreOptions options = new CoreOptions(spec.options()); + Long snapshotId = options.scanSnapshotId(); + if (snapshotId != null) { + Snapshot snapshot = table.snapshotManager().snapshot(snapshotId); + if (snapshot == null) { + throw new IllegalArgumentException("Snapshot " + snapshotId + " does not exist"); + } + return snapshot.id(); + } + String tagName = options.scanTagName(); + if (tagName != null) { + return table.tagManager().getOrThrow(tagName).id(); + } + Snapshot latest = table.snapshotManager().latestSnapshot(); + if (latest == null) { + throw new IllegalArgumentException("The prediction source has no snapshot"); + } + return latest.id(); + } + + private void execute(PredictionJob job) { + try { + if (!job.markRunning()) { + return; + } + job.executionGate.await(); + checkCanceled(); + Identifier output = executePrediction(job); + job.completeSuccess(output); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + job.completeCanceled(); + } catch (Throwable t) { + job.completeFailure( + t.getMessage() == null ? t.getClass().getSimpleName() : t.getMessage()); + } + } + + private Identifier executePrediction(PredictionJob job) throws Exception { + Identifier sourceIdentifier = job.source.toIdentifier(); + FileStoreTable source = (FileStoreTable) catalog.getTable(sourceIdentifier); + Map scanOptions = new HashMap<>(job.spec.options()); + scanOptions.remove(CoreOptions.SCAN_TAG_NAME.key()); + scanOptions.put(CoreOptions.SCAN_SNAPSHOT_ID.key(), Long.toString(job.snapshotId)); + source = source.copy(scanOptions); + + Identifier outputIdentifier = + Identifier.create( + sourceIdentifier.getDatabaseName(), + sourceIdentifier.getObjectName() + + "__" + + job.predictionId.replace('-', '_')); + Schema outputSchema = outputSchema(source.schema(), job.spec.mergeKeyColumns()); + catalog.createTable(outputIdentifier, outputSchema, false); + FileStoreTable output = (FileStoreTable) catalog.getTable(outputIdentifier); + registerOutputTable(outputIdentifier, output); + + Map fieldIndexes = fieldIndexes(source.schema()); + BatchWriteBuilder writeBuilder = output.newBatchWriteBuilder(); + ReadBuilder readBuilder = source.newReadBuilder(); + try (RecordReaderIterator rows = + new RecordReaderIterator<>( + readBuilder + .newRead() + .createReader(readBuilder.newScan().plan().splits())); + BatchTableWrite write = writeBuilder.newWrite(); + BatchTableCommit commit = writeBuilder.newCommit()) { + while (rows.hasNext()) { + checkCanceled(); + InternalRow sourceRow = rows.next(); + write.write(toOutputRow(sourceRow, source.schema(), fieldIndexes, job.spec)); + } + checkCanceled(); + commit.commit(write.prepareCommit()); + } + + Snapshot outputSnapshot = output.snapshotManager().latestSnapshot(); + server.setTableSnapshot( + outputIdentifier, + outputSnapshot, + outputSnapshot.totalRecordCount(), + 0L, + 0L, + outputSnapshot.timeMillis()); + return outputIdentifier; + } + + private void registerOutputTable(Identifier identifier, FileStoreTable table) { + TableSchema schema = table.schema(); + Map options = new HashMap<>(schema.options()); + options.put(CoreOptions.PATH.key(), catalog.getTableLocation(identifier).toString()); + tableMetadataStore.put( + identifier.getFullName(), + new TableMetadata(schema.copy(options), false, UUID.randomUUID().toString())); + } + + private static Schema outputSchema(TableSchema source, List mergeKeyColumns) { + Schema.Builder builder = Schema.newBuilder(); + Map fields = + source.fields().stream().collect(Collectors.toMap(DataField::name, field -> field)); + for (String name : mergeKeyColumns) { + builder.column(name, fields.get(name).type()); + } + return builder.column("response", DataTypes.STRING()) + .column("error", DataTypes.STRING()) + .build(); + } + + private static Map fieldIndexes(TableSchema schema) { + Map indexes = new HashMap<>(); + List fields = schema.fieldNames(); + for (int i = 0; i < fields.size(); i++) { + indexes.put(fields.get(i), i); + } + return indexes; + } + + private static InternalRow toOutputRow( + InternalRow sourceRow, + TableSchema sourceSchema, + Map fieldIndexes, + PredictionSpec spec) + throws JsonProcessingException { + Object[] values = new Object[spec.mergeKeyColumns().size() + 2]; + int outputIndex = 0; + for (String key : spec.mergeKeyColumns()) { + int sourceIndex = fieldIndexes.get(key); + DataType type = sourceSchema.fields().get(sourceIndex).type(); + values[outputIndex++] = + InternalRowUtils.copy(InternalRowUtils.get(sourceRow, sourceIndex, type), type); + } + + Map response = new LinkedHashMap<>(); + for (Map.Entry input : spec.inputs().entrySet()) { + int sourceIndex = fieldIndexes.get(input.getValue()); + DataType type = sourceSchema.fields().get(sourceIndex).type(); + Object value = InternalRowUtils.get(sourceRow, sourceIndex, type); + response.put(input.getKey(), value == null ? null : value.toString()); + } + values[outputIndex] = BinaryString.fromString(RESTApi.toJson(response)); + values[outputIndex + 1] = null; + return GenericRow.of(values); + } + + private MockResponse get(String predictionId) { + PredictionJob job = jobs.get(predictionId); + return job == null + ? notFound(predictionId) + : response(new GetPredictionResponse(job.snapshot()), 200); + } + + private MockResponse cancel(String predictionId) { + PredictionJob job = jobs.get(predictionId); + if (job == null) { + return notFound(predictionId); + } + job.cancel(); + return response(new GetPredictionResponse(job.snapshot()), 200); + } + + private MockResponse list(Map parameters) { + List matching = + jobs.values().stream() + .filter(job -> matches(job, parameters)) + .sorted(Comparator.comparing(PredictionJob::id).reversed()) + .collect(Collectors.toList()); + String pageToken = parameters.get(PAGE_TOKEN); + if (pageToken != null) { + matching = + matching.stream() + .filter(job -> job.id().compareTo(pageToken) < 0) + .collect(Collectors.toList()); + } + int maxResults = maxResults(parameters); + boolean hasMore = matching.size() > maxResults; + List page = hasMore ? matching.subList(0, maxResults) : matching; + List responses = + page.stream() + .map(job -> new GetPredictionResponse(job.snapshot())) + .collect(Collectors.toList()); + String nextPageToken = hasMore ? page.get(page.size() - 1).id() : null; + return response(new ListPredictionsResponse(responses, nextPageToken), 200); + } + + private static boolean matches(PredictionJob job, Map parameters) { + Prediction prediction = job.snapshot(); + return matches(parameters.get(DATABASE), prediction.source().database()) + && matches(parameters.get(TABLE), prediction.source().table()) + && matches(parameters.get(NAME), prediction.name()) + && matches(parameters.get(STATE), prediction.status().state().name()); + } + + private static boolean matches(@Nullable String expected, @Nullable String actual) { + return expected == null || expected.equals(actual); + } + + private static int maxResults(Map parameters) { + String value = parameters.get(MAX_RESULTS); + if (value == null) { + return RESTCatalogServer.DEFAULT_MAX_RESULTS; + } + int requested = Integer.parseInt(value); + return requested <= 0 + ? RESTCatalogServer.DEFAULT_MAX_RESULTS + : Math.min(requested, RESTCatalogServer.DEFAULT_MAX_RESULTS); + } + + private static void checkCanceled() throws InterruptedException { + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedException("Prediction was canceled"); + } + } + + private static MockResponse notFound(String predictionId) { + return error( + ErrorResponse.RESOURCE_TYPE_PREDICTION, + predictionId, + "Prediction does not exist", + 404); + } + + private static MockResponse error( + @Nullable String resourceType, + @Nullable String resourceName, + String message, + int status) { + return response(new ErrorResponse(resourceType, resourceName, message, status), status); + } + + private static MockResponse response(Object response, int status) { + try { + return new MockResponse() + .setResponseCode(status) + .setBody(RESTApi.toJson(response)) + .addHeader("Content-Type", "application/json"); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() { + executionGate.countDown(); + executor.shutdownNow(); + try { + executor.awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + private static final class PredictionJob { + + private final String predictionId; + private final PredictionSource source; + private final PredictionSpec spec; + private final long snapshotId; + private final String requestPayload; + private final CountDownLatch executionGate; + + private PredictionStatus status = + new PredictionStatus(PredictionStatus.State.PENDING, null); + private @Nullable PredictionOutput output; + private @Nullable Long startTime; + private @Nullable Long endTime; + private @Nullable Future future; + + private PredictionJob( + String predictionId, + PredictionSource source, + PredictionSpec spec, + long snapshotId, + String requestPayload, + CountDownLatch executionGate) { + this.predictionId = predictionId; + this.source = source; + this.spec = spec; + this.snapshotId = snapshotId; + this.requestPayload = requestPayload; + this.executionGate = executionGate; + } + + private synchronized String id() { + return predictionId; + } + + private synchronized void setFuture(Future future) { + this.future = future; + } + + private synchronized boolean markRunning() { + if (status.isTerminal()) { + return false; + } + status = new PredictionStatus(PredictionStatus.State.RUNNING, null); + startTime = System.currentTimeMillis(); + return true; + } + + private synchronized void completeSuccess(Identifier outputIdentifier) { + if (status.isTerminal()) { + return; + } + output = new PredictionOutput(new PredictionSource(outputIdentifier)); + status = + new PredictionStatus( + PredictionStatus.State.TERMINATED, + new PredictionStatus.TerminationDetails( + PredictionStatus.TerminationCode.SUCCESS, null)); + endTime = System.currentTimeMillis(); + } + + private synchronized void completeFailure(String message) { + if (status.isTerminal()) { + return; + } + status = + new PredictionStatus( + PredictionStatus.State.TERMINATED, + new PredictionStatus.TerminationDetails( + PredictionStatus.TerminationCode.FAILED, message)); + endTime = System.currentTimeMillis(); + } + + private synchronized void completeCanceled() { + if (status.isTerminal()) { + return; + } + status = + new PredictionStatus( + PredictionStatus.State.TERMINATED, + new PredictionStatus.TerminationDetails( + PredictionStatus.TerminationCode.CANCELED, null)); + endTime = System.currentTimeMillis(); + } + + private void cancel() { + Future task; + synchronized (this) { + if (status.isTerminal()) { + return; + } + status = new PredictionStatus(PredictionStatus.State.TERMINATING, null); + task = future; + } + if (task != null) { + task.cancel(true); + } + completeCanceled(); + } + + private synchronized Prediction snapshot() { + return new Prediction( + predictionId, + source, + spec.name(), + status, + snapshotId, + spec.model(), + output, + startTime, + endTime); + } + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index 0009d1d0287c..93a45eac4ec8 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -262,6 +262,7 @@ public class RESTCatalogServer { private final Map functionStore = new ConcurrentHashMap<>(); private final RESTCatalogModelHandler modelHandler = new RESTCatalogModelHandler(databaseStore, databaseLifecycleLocks); + private final RESTCatalogPredictionHandler predictionHandler; private final Map> columnAuthHandler = new HashMap<>(); private final Map> rowFilterAuthHandler = new HashMap<>(); private final Map> columnMaskingAuthHandler = new HashMap<>(); @@ -300,6 +301,14 @@ public RESTCatalogServer( throw new UncheckedIOException(e); } this.catalog = new RESTFileSystemCatalog(fileIO, warehousePath, catalogContext); + this.predictionHandler = + new RESTCatalogPredictionHandler( + this, + catalog, + modelHandler, + databaseStore, + tableMetadataStore, + resourcePaths.predictions()); Dispatcher dispatcher = initDispatcher(authProvider); MockWebServer mockWebServer = new MockWebServer(); mockWebServer.setDispatcher(dispatcher); @@ -322,9 +331,15 @@ public void setTableSchema(Identifier identifier, TableSchema schema) { } public void shutdown() throws IOException { + predictionHandler.close(); server.shutdown(); } + public Runnable pausePredictionExecution() { + predictionHandler.pauseExecution(); + return predictionHandler::resumeExecution; + } + public void setTableSnapshot( Identifier identifier, Snapshot snapshot, @@ -498,6 +513,10 @@ && isTableByIdRequest(request.getPath())) { } else if (StringUtils.startsWith( request.getPath(), resourcePaths.functions())) { return functionsHandle(parameters); + } else if (resourcePaths.predictions().equals(resourcePath) + || resourcePath.startsWith(resourcePaths.predictions() + "/")) { + return predictionHandler.handle( + restAuthParameter.method(), resourcePath, data, parameters); } else if (request.getPath().startsWith(databaseUri)) { String[] resources = resourcePath.substring((databaseUri + "/").length()).split("/"); From 2e0a1e22565d17b09f47abbffe52646503bf3621 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Thu, 19 Nov 2026 20:40:50 +0800 Subject: [PATCH 36/41] [core] Make prediction reference integration race safe --- .../paimon/rest/MockRESTCatalogTest.java | 153 ++++++++++- .../rest/RESTCatalogPredictionHandler.java | 245 ++++++++++++++---- .../apache/paimon/rest/RESTCatalogServer.java | 30 +++ 3 files changed, 372 insertions(+), 56 deletions(-) diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java index ab9b518efaa8..03a6bce0078b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java @@ -268,12 +268,159 @@ void testPredictionReferenceIntegration() throws Exception { Collections.emptyMap(), "reference-cancel")); waitForPredictionState(toCancel.predictionId(), PredictionStatus.State.RUNNING); - Prediction canceled = catalog.cancelPrediction(toCancel.predictionId()); - assertThat(canceled.status().state()).isEqualTo(PredictionStatus.State.TERMINATED); + Prediction canceling = catalog.cancelPrediction(toCancel.predictionId()); + assertThat(canceling.status().state()).isEqualTo(PredictionStatus.State.TERMINATING); + resumePredictionExecution.run(); + Prediction canceled = waitForPrediction(toCancel.predictionId()); assertThat(canceled.status().terminationDetails().code()) .isEqualTo(PredictionStatus.TerminationCode.CANCELED); assertThat(canceled.output()).isNull(); - resumePredictionExecution.run(); + } + + @Test + void testConcurrentIdempotentPredictionSubmissionsReturnSameJob() throws Exception { + PredictionFixture fixture = createPredictionFixture("prediction_concurrent_same"); + PredictionSpec spec = fixture.spec("same-request", "same-token"); + Runnable resumePredictionExecution = restCatalogServer.pausePredictionExecution(); + restCatalogServer.synchronizeNextPredictionSubmissions(2); + ExecutorService submissions = Executors.newFixedThreadPool(2); + try { + Future first = + submissions.submit(() -> catalog.submitPrediction(fixture.source, spec)); + Future second = + submissions.submit(() -> catalog.submitPrediction(fixture.source, spec)); + + assertThat(first.get(10, TimeUnit.SECONDS).predictionId()) + .isEqualTo(second.get(10, TimeUnit.SECONDS).predictionId()); + } finally { + resumePredictionExecution.run(); + submissions.shutdownNow(); + } + } + + @Test + void testConcurrentConflictingPredictionSubmissionsReturnConflict() throws Exception { + PredictionFixture fixture = createPredictionFixture("prediction_concurrent_conflict"); + PredictionSpec firstSpec = fixture.spec("first-request", "conflict-token"); + PredictionSpec secondSpec = fixture.spec("second-request", "conflict-token"); + Runnable resumePredictionExecution = restCatalogServer.pausePredictionExecution(); + restCatalogServer.synchronizeNextPredictionSubmissions(2); + ExecutorService submissions = Executors.newFixedThreadPool(2); + try { + List> results = + Arrays.asList( + submissions.submit( + () -> catalog.submitPrediction(fixture.source, firstSpec)), + submissions.submit( + () -> catalog.submitPrediction(fixture.source, secondSpec))); + int accepted = 0; + int conflicts = 0; + for (Future result : results) { + try { + result.get(10, TimeUnit.SECONDS); + accepted++; + } catch (ExecutionException e) { + assertThat(e.getCause()).isInstanceOf(AlreadyExistsException.class); + conflicts++; + } + } + assertThat(accepted).isEqualTo(1); + assertThat(conflicts).isEqualTo(1); + } finally { + resumePredictionExecution.run(); + submissions.shutdownNow(); + } + } + + @Test + void testCancelAfterOutputTableCreationDoesNotPublishTable() throws Exception { + PredictionFixture fixture = createPredictionFixture("prediction_cancel_after_create"); + Runnable resume = restCatalogServer.pausePredictionAfterOutputCreated(); + Prediction submitted = + catalog.submitPrediction( + fixture.source, fixture.spec("cancel-after-create", "cancel-create-token")); + + restCatalogServer.awaitPredictionOutputCreated(); + assertThat(catalog.cancelPrediction(submitted.predictionId()).status().state()) + .isEqualTo(PredictionStatus.State.TERMINATING); + resume.run(); + + assertCanceledWithoutOutputTable(fixture, submitted.predictionId()); + } + + @Test + void testCancelAfterOutputCommitDoesNotPublishTable() throws Exception { + PredictionFixture fixture = createPredictionFixture("prediction_cancel_after_commit"); + Runnable resume = restCatalogServer.pausePredictionAfterOutputCommitted(); + Prediction submitted = + catalog.submitPrediction( + fixture.source, fixture.spec("cancel-after-commit", "cancel-commit-token")); + + restCatalogServer.awaitPredictionOutputCommitted(); + assertThat(catalog.cancelPrediction(submitted.predictionId()).status().state()) + .isEqualTo(PredictionStatus.State.TERMINATING); + resume.run(); + + assertCanceledWithoutOutputTable(fixture, submitted.predictionId()); + } + + private PredictionFixture createPredictionFixture(String database) throws Exception { + Identifier source = Identifier.create(database, "clips"); + Identifier model = Identifier.create(database, "echo_model"); + catalog.createDatabase(database, false); + catalog.createTable( + source, + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("content", DataTypes.STRING()) + .build(), + false); + commitRows( + (FileStoreTable) catalog.getTable(source), + GenericRow.of(1, BinaryString.fromString("first"))); + ProviderModelDefinition definition = + new ProviderModelDefinition( + "reference", + "echo", + Collections.singletonList(new ModelField("text", "STRING", null)), + Collections.singletonList(new ModelField("response", "STRING", null)), + Collections.emptyMap()); + catalog.createModel( + model, new Model(model, "Deterministic reference model", definition), false); + return new PredictionFixture(source, model); + } + + private void assertCanceledWithoutOutputTable(PredictionFixture fixture, String predictionId) + throws Exception { + Prediction canceled = waitForPrediction(predictionId); + assertThat(canceled.status().terminationDetails().code()) + .isEqualTo(PredictionStatus.TerminationCode.CANCELED); + assertThat(canceled.output()).isNull(); + String outputName = fixture.source.getObjectName() + "__" + predictionId.replace('-', '_'); + assertThat(catalog.listTables(fixture.source.getDatabaseName())).doesNotContain(outputName); + } + + private static final class PredictionFixture { + + private final Identifier source; + private final Identifier model; + + private PredictionFixture(Identifier source, Identifier model) { + this.source = source; + this.model = model; + } + + private PredictionSpec spec(String name, String idempotencyToken) { + return new PredictionSpec( + name, + model, + Collections.singletonMap("text", "content"), + Collections.singletonList("id"), + "Echo the input", + null, + Collections.emptyMap(), + idempotencyToken); + } } private Prediction waitForPrediction(String predictionId) throws Exception { diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java index 8e7a30018e3d..11583f2c8526 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java @@ -62,9 +62,9 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; @@ -93,6 +93,9 @@ final class RESTCatalogPredictionHandler implements AutoCloseable { private final ExecutorService executor; private volatile CountDownLatch executionGate = new CountDownLatch(0); + private volatile ExecutionGate outputCreatedGate = ExecutionGate.open(); + private volatile ExecutionGate outputCommittedGate = ExecutionGate.open(); + private volatile @Nullable CyclicBarrier submissionBarrier; RESTCatalogPredictionHandler( RESTCatalogServer server, @@ -149,18 +152,39 @@ void resumeExecution() { executionGate.countDown(); } + void synchronizeNextSubmissions(int parties) { + submissionBarrier = new CyclicBarrier(parties); + } + + void pauseAfterOutputCreated() { + outputCreatedGate = ExecutionGate.paused(); + } + + void awaitOutputCreated() throws InterruptedException { + outputCreatedGate.awaitReached(); + } + + void resumeAfterOutputCreated() { + outputCreatedGate.resume(); + } + + void pauseAfterOutputCommitted() { + outputCommittedGate = ExecutionGate.paused(); + } + + void awaitOutputCommitted() throws InterruptedException { + outputCommittedGate.awaitReached(); + } + + void resumeAfterOutputCommitted() { + outputCommittedGate.resume(); + } + private MockResponse submit(String data) throws Exception { SubmitPredictionRequest request = parseRequest(data, SubmitPredictionRequest.class); PredictionJob existing = jobsByIdempotencyToken.get(request.getIdempotencyToken()); if (existing != null) { - if (existing.requestPayload.equals(data)) { - return response(new GetPredictionResponse(existing.snapshot()), 202); - } - return error( - ErrorResponse.RESOURCE_TYPE_PREDICTION, - existing.predictionId, - "idempotencyToken is already used by another prediction", - 409); + return idempotencyResponse(existing, data); } Identifier source = request.getSource().toIdentifier(); @@ -176,11 +200,45 @@ private MockResponse submit(String data) throws Exception { PredictionJob job = new PredictionJob( predictionId, request.getSource(), spec, snapshotId, data, executionGate); - jobs.put(predictionId, job); - jobsByIdempotencyToken.put(request.getIdempotencyToken(), job); - Future future = executor.submit(() -> execute(job)); - job.setFuture(future); - return response(new GetPredictionResponse(job.snapshot()), 202); + awaitSubmissionBarrier(); + PredictionJob selected = + jobsByIdempotencyToken.compute( + request.getIdempotencyToken(), + (token, current) -> { + if (current != null) { + return current; + } + jobs.put(predictionId, job); + try { + executor.submit(() -> execute(job)); + return job; + } catch (RuntimeException e) { + jobs.remove(predictionId, job); + throw e; + } + }); + return selected == job + ? response(new GetPredictionResponse(job.snapshot()), 202) + : idempotencyResponse(selected, data); + } + + private MockResponse idempotencyResponse(PredictionJob existing, String data) { + if (existing.requestPayload.equals(data)) { + return response(new GetPredictionResponse(existing.snapshot()), 202); + } + return error( + ErrorResponse.RESOURCE_TYPE_PREDICTION, + existing.predictionId, + "idempotencyToken is already used by another prediction", + 409); + } + + private void awaitSubmissionBarrier() throws Exception { + CyclicBarrier barrier = submissionBarrier; + if (barrier != null) { + barrier.await(10, TimeUnit.SECONDS); + submissionBarrier = null; + } } @Nullable @@ -275,20 +333,32 @@ private long resolveSnapshot(FileStoreTable table, PredictionSpec spec) { } private void execute(PredictionJob job) { + @Nullable Identifier preparedOutput = null; try { if (!job.markRunning()) { + job.completeCanceled(); return; } job.executionGate.await(); - checkCanceled(); - Identifier output = executePrediction(job); - job.completeSuccess(output); + checkCanceled(job); + preparedOutput = executePrediction(job); + Identifier output = preparedOutput; + if (!job.publishSuccess(output, () -> publishOutput(output))) { + cleanupOutput(output); + job.completeCanceled(); + } } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + cleanupOutput(job.stagingOutput()); job.completeCanceled(); + Thread.currentThread().interrupt(); } catch (Throwable t) { - job.completeFailure( - t.getMessage() == null ? t.getClass().getSimpleName() : t.getMessage()); + cleanupOutput(preparedOutput == null ? job.stagingOutput() : preparedOutput); + if (job.isTerminating()) { + job.completeCanceled(); + } else { + job.completeFailure( + t.getMessage() == null ? t.getClass().getSimpleName() : t.getMessage()); + } } } @@ -308,8 +378,10 @@ private Identifier executePrediction(PredictionJob job) throws Exception { + job.predictionId.replace('-', '_')); Schema outputSchema = outputSchema(source.schema(), job.spec.mergeKeyColumns()); catalog.createTable(outputIdentifier, outputSchema, false); + job.setStagingOutput(outputIdentifier); FileStoreTable output = (FileStoreTable) catalog.getTable(outputIdentifier); - registerOutputTable(outputIdentifier, output); + outputCreatedGate.reachAndAwait(); + checkCanceled(job); Map fieldIndexes = fieldIndexes(source.schema()); BatchWriteBuilder writeBuilder = output.newBatchWriteBuilder(); @@ -322,25 +394,49 @@ private Identifier executePrediction(PredictionJob job) throws Exception { BatchTableWrite write = writeBuilder.newWrite(); BatchTableCommit commit = writeBuilder.newCommit()) { while (rows.hasNext()) { - checkCanceled(); + checkCanceled(job); InternalRow sourceRow = rows.next(); write.write(toOutputRow(sourceRow, source.schema(), fieldIndexes, job.spec)); } - checkCanceled(); + checkCanceled(job); commit.commit(write.prepareCommit()); } - Snapshot outputSnapshot = output.snapshotManager().latestSnapshot(); - server.setTableSnapshot( - outputIdentifier, - outputSnapshot, - outputSnapshot.totalRecordCount(), - 0L, - 0L, - outputSnapshot.timeMillis()); + outputCommittedGate.reachAndAwait(); + checkCanceled(job); return outputIdentifier; } + private void publishOutput(Identifier identifier) { + try { + FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); + Snapshot snapshot = table.snapshotManager().latestSnapshot(); + registerOutputTable(identifier, table); + server.setTableSnapshot( + identifier, + snapshot, + snapshot.totalRecordCount(), + 0L, + 0L, + snapshot.timeMillis()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void cleanupOutput(@Nullable Identifier identifier) { + if (identifier == null) { + return; + } + tableMetadataStore.remove(identifier.getFullName()); + server.removeTableSnapshot(identifier); + try { + catalog.dropTable(identifier, true); + } catch (Exception ignored) { + // Best-effort cleanup for a test reference provider. + } + } + private void registerOutputTable(Identifier identifier, FileStoreTable table) { TableSchema schema = table.schema(); Map options = new HashMap<>(schema.options()); @@ -461,8 +557,8 @@ private static int maxResults(Map parameters) { : Math.min(requested, RESTCatalogServer.DEFAULT_MAX_RESULTS); } - private static void checkCanceled() throws InterruptedException { - if (Thread.currentThread().isInterrupted()) { + private static void checkCanceled(PredictionJob job) throws InterruptedException { + if (job.isTerminating() || Thread.currentThread().isInterrupted()) { throw new InterruptedException("Prediction was canceled"); } } @@ -497,6 +593,8 @@ private static MockResponse response(Object response, int status) { @Override public void close() { executionGate.countDown(); + outputCreatedGate.resume(); + outputCommittedGate.resume(); executor.shutdownNow(); try { executor.awaitTermination(10, TimeUnit.SECONDS); @@ -505,6 +603,40 @@ public void close() { } } + private static final class ExecutionGate { + + private final CountDownLatch reached; + private final CountDownLatch resume; + + private ExecutionGate(int count) { + this.reached = new CountDownLatch(count); + this.resume = new CountDownLatch(count); + } + + private static ExecutionGate open() { + return new ExecutionGate(0); + } + + private static ExecutionGate paused() { + return new ExecutionGate(1); + } + + private void reachAndAwait() throws InterruptedException { + reached.countDown(); + resume.await(); + } + + private void awaitReached() throws InterruptedException { + if (!reached.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("Prediction execution did not reach the expected phase"); + } + } + + private void resume() { + resume.countDown(); + } + } + private static final class PredictionJob { private final String predictionId; @@ -519,7 +651,7 @@ private static final class PredictionJob { private @Nullable PredictionOutput output; private @Nullable Long startTime; private @Nullable Long endTime; - private @Nullable Future future; + private @Nullable Identifier stagingOutput; private PredictionJob( String predictionId, @@ -540,12 +672,8 @@ private synchronized String id() { return predictionId; } - private synchronized void setFuture(Future future) { - this.future = future; - } - private synchronized boolean markRunning() { - if (status.isTerminal()) { + if (status.state() != PredictionStatus.State.PENDING) { return false; } status = new PredictionStatus(PredictionStatus.State.RUNNING, null); @@ -553,10 +681,12 @@ private synchronized boolean markRunning() { return true; } - private synchronized void completeSuccess(Identifier outputIdentifier) { - if (status.isTerminal()) { - return; + private synchronized boolean publishSuccess( + Identifier outputIdentifier, Runnable publisher) { + if (status.state() != PredictionStatus.State.RUNNING) { + return false; } + publisher.run(); output = new PredictionOutput(new PredictionSource(outputIdentifier)); status = new PredictionStatus( @@ -564,6 +694,7 @@ private synchronized void completeSuccess(Identifier outputIdentifier) { new PredictionStatus.TerminationDetails( PredictionStatus.TerminationCode.SUCCESS, null)); endTime = System.currentTimeMillis(); + return true; } private synchronized void completeFailure(String message) { @@ -590,19 +721,27 @@ private synchronized void completeCanceled() { endTime = System.currentTimeMillis(); } - private void cancel() { - Future task; - synchronized (this) { - if (status.isTerminal()) { - return; - } - status = new PredictionStatus(PredictionStatus.State.TERMINATING, null); - task = future; + private synchronized void setStagingOutput(Identifier stagingOutput) { + this.stagingOutput = stagingOutput; + } + + private synchronized @Nullable Identifier stagingOutput() { + return stagingOutput; + } + + private synchronized boolean isTerminating() { + return status.state() == PredictionStatus.State.TERMINATING; + } + + private synchronized void cancel() { + if (status.isTerminal() || status.state() == PredictionStatus.State.TERMINATING) { + return; } - if (task != null) { - task.cancel(true); + if (status.state() == PredictionStatus.State.PENDING) { + completeCanceled(); + } else { + status = new PredictionStatus(PredictionStatus.State.TERMINATING, null); } - completeCanceled(); } private synchronized Prediction snapshot() { diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index 93a45eac4ec8..659558d34bb1 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -340,6 +340,28 @@ public Runnable pausePredictionExecution() { return predictionHandler::resumeExecution; } + public void synchronizeNextPredictionSubmissions(int parties) { + predictionHandler.synchronizeNextSubmissions(parties); + } + + public Runnable pausePredictionAfterOutputCreated() { + predictionHandler.pauseAfterOutputCreated(); + return predictionHandler::resumeAfterOutputCreated; + } + + public void awaitPredictionOutputCreated() throws InterruptedException { + predictionHandler.awaitOutputCreated(); + } + + public Runnable pausePredictionAfterOutputCommitted() { + predictionHandler.pauseAfterOutputCommitted(); + return predictionHandler::resumeAfterOutputCommitted; + } + + public void awaitPredictionOutputCommitted() throws InterruptedException { + predictionHandler.awaitOutputCommitted(); + } + public void setTableSnapshot( Identifier identifier, Snapshot snapshot, @@ -355,6 +377,14 @@ public void setTableSnapshot( geTableFullNameWithSnapshotId(identifier, snapshot.id()), tableSnapshot); } + public void removeTableSnapshot(Identifier identifier) { + TableSnapshot snapshot = tableLatestSnapshotStore.remove(identifier.getFullName()); + if (snapshot != null) { + tableWithSnapshotId2SnapshotStore.remove( + geTableFullNameWithSnapshotId(identifier, snapshot.snapshot().id())); + } + } + public void setDataToken(Identifier identifier, RESTToken token) { DataTokenStore.putDataToken(warehouse, identifier.getFullName(), token); } From 36bbd5aab76afdd52e938da5a20ec5a9c826f7c9 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Thu, 19 Nov 2026 20:56:38 +0800 Subject: [PATCH 37/41] [core] Derive prediction identity from primary keys --- .../paimon/rest/MockRESTCatalogTest.java | 54 ++++++++++++++---- .../rest/RESTCatalogPredictionHandler.java | 21 +++++-- .../RESTCatalogPredictionTestControl.java | 56 +++++++++++++++++++ .../apache/paimon/rest/RESTCatalogServer.java | 35 +----------- 4 files changed, 116 insertions(+), 50 deletions(-) create mode 100644 paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTestControl.java diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java index 03a6bce0078b..0aac86063542 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java @@ -125,6 +125,7 @@ class MockRESTCatalogTest extends RESTCatalogTest { private RESTCatalogServer restCatalogServer; + private RESTCatalogPredictionTestControl predictionTestControl; private final String serverDefineHeaderName = "test-header"; private final String serverDefineHeaderValue = "test-value"; private String dataPath; @@ -223,7 +224,7 @@ void testPredictionReferenceIntegration() throws Exception { new Model(modelIdentifier, "Deterministic reference model", definition), false); - Runnable resumePredictionExecution = restCatalogServer.pausePredictionExecution(); + Runnable resumePredictionExecution = predictionTestControl.pauseExecution(); Prediction submitted = catalog.submitPrediction( sourceIdentifier, @@ -254,7 +255,7 @@ void testPredictionReferenceIntegration() throws Exception { assertThat(batchRead(catalog.getTable(completed.output().table().toIdentifier()))) .containsExactly("+I[1, {\"text\":\"first\"}, null]"); - resumePredictionExecution = restCatalogServer.pausePredictionExecution(); + resumePredictionExecution = predictionTestControl.pauseExecution(); Prediction toCancel = catalog.submitPrediction( sourceIdentifier, @@ -281,8 +282,8 @@ void testPredictionReferenceIntegration() throws Exception { void testConcurrentIdempotentPredictionSubmissionsReturnSameJob() throws Exception { PredictionFixture fixture = createPredictionFixture("prediction_concurrent_same"); PredictionSpec spec = fixture.spec("same-request", "same-token"); - Runnable resumePredictionExecution = restCatalogServer.pausePredictionExecution(); - restCatalogServer.synchronizeNextPredictionSubmissions(2); + Runnable resumePredictionExecution = predictionTestControl.pauseExecution(); + predictionTestControl.synchronizeNextSubmissions(2); ExecutorService submissions = Executors.newFixedThreadPool(2); try { Future first = @@ -303,8 +304,8 @@ void testConcurrentConflictingPredictionSubmissionsReturnConflict() throws Excep PredictionFixture fixture = createPredictionFixture("prediction_concurrent_conflict"); PredictionSpec firstSpec = fixture.spec("first-request", "conflict-token"); PredictionSpec secondSpec = fixture.spec("second-request", "conflict-token"); - Runnable resumePredictionExecution = restCatalogServer.pausePredictionExecution(); - restCatalogServer.synchronizeNextPredictionSubmissions(2); + Runnable resumePredictionExecution = predictionTestControl.pauseExecution(); + predictionTestControl.synchronizeNextSubmissions(2); ExecutorService submissions = Executors.newFixedThreadPool(2); try { List> results = @@ -335,12 +336,12 @@ void testConcurrentConflictingPredictionSubmissionsReturnConflict() throws Excep @Test void testCancelAfterOutputTableCreationDoesNotPublishTable() throws Exception { PredictionFixture fixture = createPredictionFixture("prediction_cancel_after_create"); - Runnable resume = restCatalogServer.pausePredictionAfterOutputCreated(); + Runnable resume = predictionTestControl.pauseAfterOutputCreated(); Prediction submitted = catalog.submitPrediction( fixture.source, fixture.spec("cancel-after-create", "cancel-create-token")); - restCatalogServer.awaitPredictionOutputCreated(); + predictionTestControl.awaitOutputCreated(); assertThat(catalog.cancelPrediction(submitted.predictionId()).status().state()) .isEqualTo(PredictionStatus.State.TERMINATING); resume.run(); @@ -351,12 +352,12 @@ void testCancelAfterOutputTableCreationDoesNotPublishTable() throws Exception { @Test void testCancelAfterOutputCommitDoesNotPublishTable() throws Exception { PredictionFixture fixture = createPredictionFixture("prediction_cancel_after_commit"); - Runnable resume = restCatalogServer.pausePredictionAfterOutputCommitted(); + Runnable resume = predictionTestControl.pauseAfterOutputCommitted(); Prediction submitted = catalog.submitPrediction( fixture.source, fixture.spec("cancel-after-commit", "cancel-commit-token")); - restCatalogServer.awaitPredictionOutputCommitted(); + predictionTestControl.awaitOutputCommitted(); assertThat(catalog.cancelPrediction(submitted.predictionId()).status().state()) .isEqualTo(PredictionStatus.State.TERMINATING); resume.run(); @@ -364,6 +365,22 @@ void testCancelAfterOutputCommitDoesNotPublishTable() throws Exception { assertCanceledWithoutOutputTable(fixture, submitted.predictionId()); } + @Test + void testPredictionDerivesMergeKeyFromPrimaryKey() throws Exception { + PredictionFixture fixture = createPredictionFixture("prediction_derived_merge_key"); + Prediction submitted = + catalog.submitPrediction( + fixture.source, + fixture.specWithoutMergeKey("derived-merge-key", "derived-key-token")); + + Prediction completed = waitForPrediction(submitted.predictionId()); + assertThat(completed.status().terminationDetails().code()) + .isEqualTo(PredictionStatus.TerminationCode.SUCCESS); + assertThat(batchRead(catalog.getTable(completed.output().table().toIdentifier()))) + .containsExactlyInAnyOrder( + "+I[1, {\"text\":\"first\"}, null]", "+I[2, {\"text\":\"second\"}, null]"); + } + private PredictionFixture createPredictionFixture(String database) throws Exception { Identifier source = Identifier.create(database, "clips"); Identifier model = Identifier.create(database, "echo_model"); @@ -373,11 +390,14 @@ private PredictionFixture createPredictionFixture(String database) throws Except Schema.newBuilder() .column("id", DataTypes.INT()) .column("content", DataTypes.STRING()) + .primaryKey("id") + .option(CoreOptions.BUCKET.key(), "1") .build(), false); commitRows( (FileStoreTable) catalog.getTable(source), - GenericRow.of(1, BinaryString.fromString("first"))); + GenericRow.of(1, BinaryString.fromString("first")), + GenericRow.of(2, BinaryString.fromString("second"))); ProviderModelDefinition definition = new ProviderModelDefinition( "reference", @@ -411,11 +431,20 @@ private PredictionFixture(Identifier source, Identifier model) { } private PredictionSpec spec(String name, String idempotencyToken) { + return spec(name, Collections.singletonList("id"), idempotencyToken); + } + + private PredictionSpec specWithoutMergeKey(String name, String idempotencyToken) { + return spec(name, null, idempotencyToken); + } + + private PredictionSpec spec( + String name, List mergeKeyColumns, String idempotencyToken) { return new PredictionSpec( name, model, Collections.singletonMap("text", "content"), - Collections.singletonList("id"), + mergeKeyColumns, "Echo the input", null, Collections.emptyMap(), @@ -2259,6 +2288,7 @@ private RESTCatalog initCatalogUtil( this.config = new ConfigResponse(defaultConf, ImmutableMap.of()); restCatalogServer = new RESTCatalogServer(dataPath, this.authProvider, this.config, restWarehouse); + predictionTestControl = restCatalogServer.predictionTestControl(); restCatalogServer.start(); for (Map.Entry entry : this.authMap.entrySet()) { options.set(entry.getKey(), entry.getValue()); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java index 11583f2c8526..c1bca95c8710 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java @@ -376,7 +376,8 @@ private Identifier executePrediction(PredictionJob job) throws Exception { sourceIdentifier.getObjectName() + "__" + job.predictionId.replace('-', '_')); - Schema outputSchema = outputSchema(source.schema(), job.spec.mergeKeyColumns()); + List mergeKeyColumns = mergeKeyColumns(source.schema(), job.spec); + Schema outputSchema = outputSchema(source.schema(), mergeKeyColumns); catalog.createTable(outputIdentifier, outputSchema, false); job.setStagingOutput(outputIdentifier); FileStoreTable output = (FileStoreTable) catalog.getTable(outputIdentifier); @@ -396,7 +397,13 @@ private Identifier executePrediction(PredictionJob job) throws Exception { while (rows.hasNext()) { checkCanceled(job); InternalRow sourceRow = rows.next(); - write.write(toOutputRow(sourceRow, source.schema(), fieldIndexes, job.spec)); + write.write( + toOutputRow( + sourceRow, + source.schema(), + fieldIndexes, + mergeKeyColumns, + job.spec)); } checkCanceled(job); commit.commit(write.prepareCommit()); @@ -429,7 +436,6 @@ private void cleanupOutput(@Nullable Identifier identifier) { return; } tableMetadataStore.remove(identifier.getFullName()); - server.removeTableSnapshot(identifier); try { catalog.dropTable(identifier, true); } catch (Exception ignored) { @@ -458,6 +464,10 @@ private static Schema outputSchema(TableSchema source, List mergeKeyColu .build(); } + private static List mergeKeyColumns(TableSchema source, PredictionSpec spec) { + return spec.mergeKeyColumns().isEmpty() ? source.primaryKeys() : spec.mergeKeyColumns(); + } + private static Map fieldIndexes(TableSchema schema) { Map indexes = new HashMap<>(); List fields = schema.fieldNames(); @@ -471,11 +481,12 @@ private static InternalRow toOutputRow( InternalRow sourceRow, TableSchema sourceSchema, Map fieldIndexes, + List mergeKeyColumns, PredictionSpec spec) throws JsonProcessingException { - Object[] values = new Object[spec.mergeKeyColumns().size() + 2]; + Object[] values = new Object[mergeKeyColumns.size() + 2]; int outputIndex = 0; - for (String key : spec.mergeKeyColumns()) { + for (String key : mergeKeyColumns) { int sourceIndex = fieldIndexes.get(key); DataType type = sourceSchema.fields().get(sourceIndex).type(); values[outputIndex++] = diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTestControl.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTestControl.java new file mode 100644 index 000000000000..3afc3ec25b29 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTestControl.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +/** Controls prediction execution phases for deterministic REST integration tests. */ +final class RESTCatalogPredictionTestControl { + + private final RESTCatalogPredictionHandler handler; + + RESTCatalogPredictionTestControl(RESTCatalogPredictionHandler handler) { + this.handler = handler; + } + + Runnable pauseExecution() { + handler.pauseExecution(); + return handler::resumeExecution; + } + + void synchronizeNextSubmissions(int parties) { + handler.synchronizeNextSubmissions(parties); + } + + Runnable pauseAfterOutputCreated() { + handler.pauseAfterOutputCreated(); + return handler::resumeAfterOutputCreated; + } + + void awaitOutputCreated() throws InterruptedException { + handler.awaitOutputCreated(); + } + + Runnable pauseAfterOutputCommitted() { + handler.pauseAfterOutputCommitted(); + return handler::resumeAfterOutputCommitted; + } + + void awaitOutputCommitted() throws InterruptedException { + handler.awaitOutputCommitted(); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index 659558d34bb1..847456a2a993 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -335,31 +335,8 @@ public void shutdown() throws IOException { server.shutdown(); } - public Runnable pausePredictionExecution() { - predictionHandler.pauseExecution(); - return predictionHandler::resumeExecution; - } - - public void synchronizeNextPredictionSubmissions(int parties) { - predictionHandler.synchronizeNextSubmissions(parties); - } - - public Runnable pausePredictionAfterOutputCreated() { - predictionHandler.pauseAfterOutputCreated(); - return predictionHandler::resumeAfterOutputCreated; - } - - public void awaitPredictionOutputCreated() throws InterruptedException { - predictionHandler.awaitOutputCreated(); - } - - public Runnable pausePredictionAfterOutputCommitted() { - predictionHandler.pauseAfterOutputCommitted(); - return predictionHandler::resumeAfterOutputCommitted; - } - - public void awaitPredictionOutputCommitted() throws InterruptedException { - predictionHandler.awaitOutputCommitted(); + RESTCatalogPredictionTestControl predictionTestControl() { + return new RESTCatalogPredictionTestControl(predictionHandler); } public void setTableSnapshot( @@ -377,14 +354,6 @@ public void setTableSnapshot( geTableFullNameWithSnapshotId(identifier, snapshot.id()), tableSnapshot); } - public void removeTableSnapshot(Identifier identifier) { - TableSnapshot snapshot = tableLatestSnapshotStore.remove(identifier.getFullName()); - if (snapshot != null) { - tableWithSnapshotId2SnapshotStore.remove( - geTableFullNameWithSnapshotId(identifier, snapshot.snapshot().id())); - } - } - public void setDataToken(Identifier identifier, RESTToken token) { DataTokenStore.putDataToken(warehouse, identifier.getFullName(), token); } From a163bec669d7b8728161d81343bbe2d0b4b298aa Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Thu, 19 Nov 2026 21:20:33 +0800 Subject: [PATCH 38/41] [core] Validate derived prediction merge keys --- .../paimon/rest/MockRESTCatalogTest.java | 24 +++++++++++++++++-- .../rest/RESTCatalogPredictionHandler.java | 2 +- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java index 0aac86063542..3c5370e259da 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java @@ -381,16 +381,36 @@ void testPredictionDerivesMergeKeyFromPrimaryKey() throws Exception { "+I[1, {\"text\":\"first\"}, null]", "+I[2, {\"text\":\"second\"}, null]"); } + @Test + void testPredictionRejectsDerivedMergeKeyUsingReservedOutputName() throws Exception { + PredictionFixture fixture = + createPredictionFixture("prediction_reserved_derived_key", "response"); + + assertThatThrownBy( + () -> + catalog.submitPrediction( + fixture.source, + fixture.specWithoutMergeKey( + "reserved-derived-key", "reserved-key-token"))) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("reserved output name"); + } + private PredictionFixture createPredictionFixture(String database) throws Exception { + return createPredictionFixture(database, "id"); + } + + private PredictionFixture createPredictionFixture(String database, String primaryKey) + throws Exception { Identifier source = Identifier.create(database, "clips"); Identifier model = Identifier.create(database, "echo_model"); catalog.createDatabase(database, false); catalog.createTable( source, Schema.newBuilder() - .column("id", DataTypes.INT()) + .column(primaryKey, DataTypes.INT()) .column("content", DataTypes.STRING()) - .primaryKey("id") + .primaryKey(primaryKey) .option(CoreOptions.BUCKET.key(), "1") .build(), false); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java index c1bca95c8710..d18a0c693b10 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java @@ -296,7 +296,7 @@ private MockResponse validate(Identifier source, PredictionSpec spec) { 404); } } - for (String key : spec.mergeKeyColumns()) { + for (String key : mergeKeyColumns(schema, spec)) { if (!fields.contains(key)) { return error( ErrorResponse.RESOURCE_TYPE_COLUMN, From 7f2561d1d82d71fdfe54290479016919a1d0d211 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Thu, 19 Nov 2026 21:37:31 +0800 Subject: [PATCH 39/41] [docs] Clarify prediction get operation --- docs/docs/concepts/rest/rest-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/concepts/rest/rest-api.md b/docs/docs/concepts/rest/rest-api.md index 286ba4059917..c5c49b4c660b 100644 --- a/docs/docs/concepts/rest/rest-api.md +++ b/docs/docs/concepts/rest/rest-api.md @@ -55,7 +55,7 @@ payloads, and error responses are defined in the OpenAPI specification. | Branches and tags | Manage named histories and retained snapshots. | Table-scoped `branches` and `tags`. | | Consumers | List and reset streaming consumer progress. | Table-scoped `consumers`. | | Views and functions | Manage reusable SQL and function definitions. | Database- and catalog-scoped `views` and `functions`. | -| Prediction jobs | Submit, list, inspect, and cancel asynchronous predictions. | Catalog-scoped `predictions`; the source table is part of the submission. | +| Prediction jobs | Submit, list, get, and cancel asynchronous predictions. | Catalog-scoped `predictions`; the source table is part of the submission. | In this table, **table-scoped** means `/v1/{prefix}/databases/{database}/tables/{table}`. Catalog-wide listing and detail-listing From f785f4e6f1eae9e1ac21709e0ae5185de3ee9a9c Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Fri, 20 Nov 2026 17:31:23 +0800 Subject: [PATCH 40/41] fix: harden prediction reference execution --- .../paimon/rest/MockRESTCatalogTest.java | 51 ++++++++++++++++++- .../rest/RESTCatalogPredictionHandler.java | 33 +++++++++++- 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java index 3c5370e259da..20ea9feb9d76 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java @@ -77,6 +77,7 @@ import org.apache.paimon.table.sink.BatchTableCommit; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowKind; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.BuildVersions; import org.apache.paimon.utils.InstantiationUtil; @@ -396,12 +397,60 @@ void testPredictionRejectsDerivedMergeKeyUsingReservedOutputName() throws Except .hasMessageContaining("reserved output name"); } + @Test + void testPredictionRejectsRowFilteringScanOptions() throws Exception { + PredictionFixture fixture = createPredictionFixture("prediction_scan_options", "id", 2); + PredictionSpec spec = + new PredictionSpec( + "restricted-scan-option", + fixture.model, + Collections.singletonMap("text", "content"), + Collections.singletonList("id"), + "Echo the input", + null, + Collections.singletonMap(CoreOptions.SCAN_BUCKET.key(), "0"), + "restricted-scan-option-token"); + + assertThatThrownBy(() -> catalog.submitPrediction(fixture.source, spec)) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining(CoreOptions.SCAN_BUCKET.key()); + } + + @Test + void testPredictionPublishesEmptyResultTable() throws Exception { + PredictionFixture fixture = createPredictionFixture("prediction_empty_result"); + FileStoreTable source = (FileStoreTable) catalog.getTable(fixture.source); + commitRows( + source, + GenericRow.ofKind(RowKind.DELETE, 1, BinaryString.fromString("first")), + GenericRow.ofKind(RowKind.DELETE, 2, BinaryString.fromString("second"))); + assertThat(batchRead(source)).isEmpty(); + + Prediction submitted = + catalog.submitPrediction( + fixture.source, fixture.spec("empty-result", "empty-result-token")); + Prediction completed = waitForPrediction(submitted.predictionId()); + + assertThat(completed.status().terminationDetails().code()) + .isEqualTo(PredictionStatus.TerminationCode.SUCCESS); + FileStoreTable output = + (FileStoreTable) catalog.getTable(completed.output().table().toIdentifier()); + assertThat(output.snapshotManager().latestSnapshot()).isNotNull(); + assertThat(output.snapshotManager().latestSnapshot().totalRecordCount()).isZero(); + assertThat(batchRead(output)).isEmpty(); + } + private PredictionFixture createPredictionFixture(String database) throws Exception { return createPredictionFixture(database, "id"); } private PredictionFixture createPredictionFixture(String database, String primaryKey) throws Exception { + return createPredictionFixture(database, primaryKey, 1); + } + + private PredictionFixture createPredictionFixture( + String database, String primaryKey, int buckets) throws Exception { Identifier source = Identifier.create(database, "clips"); Identifier model = Identifier.create(database, "echo_model"); catalog.createDatabase(database, false); @@ -411,7 +460,7 @@ private PredictionFixture createPredictionFixture(String database, String primar .column(primaryKey, DataTypes.INT()) .column("content", DataTypes.STRING()) .primaryKey(primaryKey) - .option(CoreOptions.BUCKET.key(), "1") + .option(CoreOptions.BUCKET.key(), Integer.toString(buckets)) .build(), false); commitRows( diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java index d18a0c693b10..de4588baffb2 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java @@ -54,6 +54,7 @@ import javax.annotation.Nullable; +import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashMap; @@ -287,6 +288,15 @@ private MockResponse validate(Identifier source, PredictionSpec spec) { 404); } List fields = schema.fieldNames(); + List unsupportedScanOptions = + unsupportedScanOptions(schema.options(), spec.options()); + if (!unsupportedScanOptions.isEmpty()) { + return error( + null, + null, + "Prediction does not support scan options: " + unsupportedScanOptions, + 400); + } for (String input : spec.inputs().values()) { if (!fields.contains(input)) { return error( @@ -453,7 +463,8 @@ private void registerOutputTable(Identifier identifier, FileStoreTable table) { } private static Schema outputSchema(TableSchema source, List mergeKeyColumns) { - Schema.Builder builder = Schema.newBuilder(); + Schema.Builder builder = + Schema.newBuilder().option(CoreOptions.SNAPSHOT_IGNORE_EMPTY_COMMIT.key(), "false"); Map fields = source.fields().stream().collect(Collectors.toMap(DataField::name, field -> field)); for (String name : mergeKeyColumns) { @@ -464,6 +475,26 @@ private static Schema outputSchema(TableSchema source, List mergeKeyColu .build(); } + private static List unsupportedScanOptions( + Map tableOptions, Map requestOptions) { + List unsupported = new ArrayList<>(); + collectUnsupportedScanOptions(tableOptions, unsupported); + collectUnsupportedScanOptions(requestOptions, unsupported); + return unsupported; + } + + private static void collectUnsupportedScanOptions( + Map options, List unsupported) { + for (String key : options.keySet()) { + if ((key.startsWith("scan.") || key.startsWith("incremental-")) + && !CoreOptions.SCAN_SNAPSHOT_ID.key().equals(key) + && !CoreOptions.SCAN_TAG_NAME.key().equals(key) + && !unsupported.contains(key)) { + unsupported.add(key); + } + } + } + private static List mergeKeyColumns(TableSchema source, PredictionSpec spec) { return spec.mergeKeyColumns().isEmpty() ? source.primaryKeys() : spec.mergeKeyColumns(); } From f84bfd93ff4dd2961d7277b4b637c3ede8446501 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Fri, 20 Nov 2026 18:05:02 +0800 Subject: [PATCH 41/41] fix: reject incompatible prediction scan modes --- .../paimon/rest/MockRESTCatalogTest.java | 65 ++++++++++++++----- .../rest/RESTCatalogPredictionHandler.java | 4 +- paimon-python/pypaimon/multimodal/table.py | 13 ++-- .../pypaimon/tests/rest/prediction_test.py | 23 +++++++ 4 files changed, 82 insertions(+), 23 deletions(-) diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java index 20ea9feb9d76..bdec50a83960 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java @@ -97,6 +97,7 @@ import java.net.URI; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.AbstractMap; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -398,22 +399,45 @@ void testPredictionRejectsDerivedMergeKeyUsingReservedOutputName() throws Except } @Test - void testPredictionRejectsRowFilteringScanOptions() throws Exception { + void testPredictionRejectsUnsupportedScanOptions() throws Exception { PredictionFixture fixture = createPredictionFixture("prediction_scan_options", "id", 2); - PredictionSpec spec = - new PredictionSpec( - "restricted-scan-option", - fixture.model, - Collections.singletonMap("text", "content"), - Collections.singletonList("id"), - "Echo the input", - null, - Collections.singletonMap(CoreOptions.SCAN_BUCKET.key(), "0"), - "restricted-scan-option-token"); + for (Map.Entry option : + Arrays.asList( + new AbstractMap.SimpleEntry<>(CoreOptions.SCAN_BUCKET.key(), "0"), + new AbstractMap.SimpleEntry<>( + CoreOptions.STREAM_SCAN_MODE.key(), "file-monitor"))) { + PredictionSpec spec = + new PredictionSpec( + "restricted-" + option.getKey(), + fixture.model, + Collections.singletonMap("text", "content"), + Collections.singletonList("id"), + "Echo the input", + null, + Collections.singletonMap(option.getKey(), option.getValue()), + "restricted-" + option.getKey() + "-token"); + + assertThatThrownBy(() -> catalog.submitPrediction(fixture.source, spec)) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining(option.getKey()); + } - assertThatThrownBy(() -> catalog.submitPrediction(fixture.source, spec)) + PredictionFixture configured = + createPredictionFixture( + "prediction_stream_scan_mode", + "id", + 1, + Collections.singletonMap( + CoreOptions.STREAM_SCAN_MODE.key(), "file-monitor")); + assertThatThrownBy( + () -> + catalog.submitPrediction( + configured.source, + configured.spec( + "configured-stream-scan-mode", + "configured-stream-scan-mode-token"))) .isInstanceOf(BadRequestException.class) - .hasMessageContaining(CoreOptions.SCAN_BUCKET.key()); + .hasMessageContaining(CoreOptions.STREAM_SCAN_MODE.key()); } @Test @@ -451,18 +475,23 @@ private PredictionFixture createPredictionFixture(String database, String primar private PredictionFixture createPredictionFixture( String database, String primaryKey, int buckets) throws Exception { + return createPredictionFixture(database, primaryKey, buckets, Collections.emptyMap()); + } + + private PredictionFixture createPredictionFixture( + String database, String primaryKey, int buckets, Map tableOptions) + throws Exception { Identifier source = Identifier.create(database, "clips"); Identifier model = Identifier.create(database, "echo_model"); catalog.createDatabase(database, false); - catalog.createTable( - source, + Schema.Builder schema = Schema.newBuilder() .column(primaryKey, DataTypes.INT()) .column("content", DataTypes.STRING()) .primaryKey(primaryKey) - .option(CoreOptions.BUCKET.key(), Integer.toString(buckets)) - .build(), - false); + .option(CoreOptions.BUCKET.key(), Integer.toString(buckets)); + tableOptions.forEach(schema::option); + catalog.createTable(source, schema.build(), false); commitRows( (FileStoreTable) catalog.getTable(source), GenericRow.of(1, BinaryString.fromString("first")), diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java index de4588baffb2..d1e8a860501d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java @@ -486,7 +486,9 @@ private static List unsupportedScanOptions( private static void collectUnsupportedScanOptions( Map options, List unsupported) { for (String key : options.keySet()) { - if ((key.startsWith("scan.") || key.startsWith("incremental-")) + if ((key.startsWith("scan.") + || key.startsWith("incremental-") + || CoreOptions.STREAM_SCAN_MODE.key().equals(key)) && !CoreOptions.SCAN_SNAPSHOT_ID.key().equals(key) && !CoreOptions.SCAN_TAG_NAME.key().equals(key) && !unsupported.contains(key)) { diff --git a/paimon-python/pypaimon/multimodal/table.py b/paimon-python/pypaimon/multimodal/table.py index 801500f6f0cf..cee8c5f474f1 100644 --- a/paimon-python/pypaimon/multimodal/table.py +++ b/paimon-python/pypaimon/multimodal/table.py @@ -142,15 +142,20 @@ def predict( table_options = self.raw_table.options.options.to_map() allowed = {"scan.snapshot-id", "scan.tag-name"} prediction_options = dict(options or {}) + + def unsupported_scan_option(key): + return key not in allowed and ( + key.startswith("scan.") + or key.startswith("incremental-") + or key == "stream-scan-mode") + unsupported = [ key for key in table_options - if (key.startswith("scan.") or key.startswith("incremental-")) - and key not in allowed + if unsupported_scan_option(key) ] unsupported.extend([ key for key in prediction_options - if (key.startswith("scan.") or key.startswith("incremental-")) - and key not in allowed + if unsupported_scan_option(key) ]) if unsupported: raise ValueError( diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py index e1edce283218..c85c03fb3b63 100644 --- a/paimon-python/pypaimon/tests/rest/prediction_test.py +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -284,6 +284,29 @@ def test_table_submit_fixes_view_and_returns_waiter(self): self.assertEqual("retry-1", sent.idempotency_token) raw.new_read_builder.assert_not_called() + def test_table_submit_rejects_stream_scan_mode(self): + for table_options, request_options in ( + ({"stream-scan-mode": "file-monitor"}, None), + ({}, {"stream-scan-mode": "file-monitor"}), + ): + with self.subTest( + table_options=table_options, + request_options=request_options): + raw = Mock(spec=FileStoreTable) + raw.options = CoreOptions(Options(table_options)) + raw.identifier = Identifier.create("quality", "clips") + catalog = Mock() + table = MultimodalTable(catalog, "quality.clips", raw) + + with self.assertRaisesRegex(ValueError, "stream-scan-mode"): + table.predict( + model="quality.qwen_vl", + inputs={"video": "content"}, + instructions="Classify", + options=request_options, + ) + catalog.submit_prediction.assert_not_called() + def test_table_queries_return_metadata_and_cancel_returns_waiter(self): raw = Mock(spec=FileStoreTable) raw.identifier = Identifier.create("quality", "clips")