diff --git a/docs/docs/concepts/rest/rest-api.md b/docs/docs/concepts/rest/rest-api.md index b3279b265231..c5c49b4c660b 100644 --- a/docs/docs/concepts/rest/rest-api.md +++ b/docs/docs/concepts/rest/rest-api.md @@ -55,6 +55,7 @@ payloads, and error responses are defined in the OpenAPI specification. | Branches and tags | Manage named histories and retained snapshots. | Table-scoped `branches` and `tags`. | | Consumers | List and reset streaming consumer progress. | Table-scoped `consumers`. | | Views and functions | Manage reusable SQL and function definitions. | Database- and catalog-scoped `views` and `functions`. | +| Prediction jobs | Submit, list, get, and cancel asynchronous predictions. | Catalog-scoped `predictions`; the source table is part of the submission. | In this table, **table-scoped** means `/v1/{prefix}/databases/{database}/tables/{table}`. Catalog-wide listing and detail-listing diff --git a/docs/static/rest-catalog-open-api.yaml b/docs/static/rest-catalog-open-api.yaml index 40d97b3c1c7f..6791eccc76e6 100644 --- a/docs/static/rest-catalog-open-api.yaml +++ b/docs/static/rest-catalog-open-api.yaml @@ -133,6 +133,102 @@ paths: "404": $ref: '#/components/responses/DatabaseOrModelNotExistErrorResponse' "501": {description: Models are not supported} + /v1/{prefix}/predictions: + post: + tags: [prediction] + summary: Submit a prediction job + operationId: submitPrediction + description: >- + Records the source snapshot and resolves the registered model. Execution reads + the recorded snapshot and fails if it is no longer available. + Reusing an idempotencyToken with the same payload returns the existing job; + conflicting reuse returns 409. + parameters: + - {name: prefix, in: path, required: true, schema: {type: string}} + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/SubmitPredictionRequest' + responses: + "202": + description: Prediction accepted + content: + application/json: + schema: + $ref: '#/components/schemas/GetPredictionResponse' + "400": + $ref: '#/components/responses/BadRequestErrorResponse' + "404": + $ref: '#/components/responses/TableOrModelNotExistErrorResponse' + "409": + $ref: '#/components/responses/ResourceAlreadyExistErrorResponse' + "501": {description: Predictions are not supported} + get: + tags: [prediction] + summary: List prediction jobs + operationId: listPredictions + description: Newest first. + parameters: + - {name: prefix, in: path, required: true, schema: {type: string}} + - {name: database, in: query, required: false, schema: {type: string}} + - {name: table, in: query, required: false, schema: {type: string}} + - {name: name, in: query, required: false, schema: {type: string}} + - name: state + in: query + required: false + schema: + $ref: '#/components/schemas/PredictionState' + - {name: maxResults, in: query, required: false, schema: {type: integer, format: int32, minimum: 1}} + - {name: pageToken, in: query, required: false, schema: {type: string}} + responses: + "200": + description: Prediction jobs + content: + application/json: + schema: + $ref: '#/components/schemas/ListPredictionsResponse' + "501": {description: Predictions are not supported} + /v1/{prefix}/predictions/{predictionId}: + get: + tags: [prediction] + summary: Get a prediction job + operationId: getPrediction + parameters: + - {name: prefix, in: path, required: true, schema: {type: string}} + - {name: predictionId, in: path, required: true, schema: {$ref: '#/components/schemas/PredictionId'}} + responses: + "200": + description: Prediction metadata + content: + application/json: + schema: + $ref: '#/components/schemas/GetPredictionResponse' + "404": + $ref: '#/components/responses/PredictionNotExistErrorResponse' + "501": {description: Predictions are not supported} + /v1/{prefix}/predictions/{predictionId}/cancel: + post: + tags: [prediction] + summary: Cancel a prediction job + operationId: cancelPrediction + description: >- + Requests cancellation and returns the latest state. Terminal jobs are + unchanged. + parameters: + - {name: prefix, in: path, required: true, schema: {type: string}} + - {name: predictionId, in: path, required: true, schema: {$ref: '#/components/schemas/PredictionId'}} + responses: + "200": + description: Latest prediction metadata + content: + application/json: + schema: + $ref: '#/components/schemas/GetPredictionResponse' + "404": + $ref: '#/components/responses/PredictionNotExistErrorResponse' + "501": {description: Predictions are not supported} /v1/config: get: tags: @@ -2881,6 +2977,32 @@ components: resourceType: MODEL resourceName: model code: 404 + TableOrModelNotExistErrorResponse: + description: Database, table, or model not found + content: + application/json: + schema: + allOf: + - $ref: '#/components/schemas/ErrorResponse' + - type: object + required: [ resourceType ] + properties: + resourceType: + type: string + enum: [ "DATABASE", "TABLE", "MODEL" ] + PredictionNotExistErrorResponse: + description: Prediction not found + content: + application/json: + schema: + allOf: + - $ref: '#/components/schemas/ErrorResponse' + - type: object + required: [ resourceType ] + properties: + resourceType: + type: string + enum: [ "PREDICTION" ] TableNotExistErrorResponse: description: Not Found - TableNotExistException, the table does not exist @@ -3264,7 +3386,7 @@ components: type: string resourceType: type: [ string, "null" ] - enum: [ "DATABASE", "TABLE", "PARTITION", "COLUMN", "SNAPSHOT", "BRANCH", "TAG", "VIEW", "SEMANTIC_VIEW", "DIALECT", "FUNCTION", "MODEL", "DEFINITION", "UNKNOWN", null ] + enum: [ "DATABASE", "TABLE", "PARTITION", "COLUMN", "SNAPSHOT", "BRANCH", "TAG", "VIEW", "SEMANTIC_VIEW", "DIALECT", "FUNCTION", "MODEL", "PREDICTION", "DEFINITION", "UNKNOWN", null ] resourceName: type: [ string, "null" ] code: @@ -4467,6 +4589,12 @@ components: minLength: 1 pattern: '^(?!\.{1,2}$).+' description: A non-empty path segment other than `.` or `..`. + PredictionId: + type: string + minLength: 1 + maxLength: 64 + pattern: '^(?!\.{1,2}$).+' + description: A prediction identifier other than `.` or `..`. ModelField: type: object additionalProperties: false @@ -4548,6 +4676,158 @@ components: $ref: '#/components/schemas/GetModelResponse' nextPageToken: type: [ string, "null" ] + PredictionState: + type: string + enum: [PENDING, RUNNING, TERMINATING, TERMINATED] + PredictionTerminationCode: + type: string + enum: [SUCCESS, SUCCESS_WITH_FAILURES, FAILED, CANCELED] + PredictionTerminationDetails: + type: object + additionalProperties: false + required: [code] + properties: + code: + $ref: '#/components/schemas/PredictionTerminationCode' + message: + type: string + PredictionStatus: + type: object + additionalProperties: false + required: [state] + properties: + state: + $ref: '#/components/schemas/PredictionState' + terminationDetails: + $ref: '#/components/schemas/PredictionTerminationDetails' + oneOf: + - properties: + state: {enum: [PENDING, RUNNING]} + not: + required: [terminationDetails] + - properties: + state: {const: TERMINATING} + - properties: + state: {const: TERMINATED} + required: [terminationDetails] + description: >- + terminationDetails is forbidden for PENDING and RUNNING, optional for + TERMINATING, and required for TERMINATED. + PredictionSource: + type: object + additionalProperties: false + required: [database, table] + properties: + database: {type: string, minLength: 1, maxLength: 128} + table: {type: string, minLength: 1, maxLength: 128} + PredictionOutput: + type: object + additionalProperties: false + required: [table] + properties: + table: + $ref: '#/components/schemas/PredictionSource' + description: >- + Managed result table containing source identity columns, a response string, + and an error string. + SubmitPredictionRequest: + type: object + additionalProperties: false + required: [source, model, inputs, instructions] + properties: + source: + $ref: '#/components/schemas/PredictionSource' + name: + type: string + minLength: 1 + maxLength: 128 + description: Optional non-unique label. + idempotencyToken: + type: string + minLength: 1 + maxLength: 64 + description: >- + Optional token reused for retries of one logical submission. SDKs generate + one by default; requests without a token are independent submissions. + model: + allOf: + - $ref: '#/components/schemas/Identifier' + required: [database, object] + properties: + database: {type: string, minLength: 1, maxLength: 128} + object: {type: string, minLength: 1, maxLength: 128} + inputs: + type: object + minProperties: 1 + propertyNames: {type: string, minLength: 1, maxLength: 128} + additionalProperties: + type: string + minLength: 1 + maxLength: 1024 + description: Model input-to-column mappings. + mergeKeyColumns: + type: array + uniqueItems: true + items: {type: string, minLength: 1, maxLength: 128} + description: >- + Stable source columns used when table primary keys or row IDs are unavailable. + Omit to let the server derive identity; without stable identity the prediction + runs as a full refresh. + instructions: + type: string + minLength: 1 + maxLength: 1048576 + description: Natural-language guidance for the prediction. + where: + type: string + minLength: 1 + maxLength: 1048576 + options: + type: object + propertyNames: {type: string, minLength: 1, maxLength: 128} + additionalProperties: + type: string + minLength: 1 + maxLength: 1024 + description: >- + Paimon dynamic options. Use scan.snapshot-id or scan.tag-name for time + travel; the two options are mutually exclusive. + GetPredictionResponse: + type: object + additionalProperties: false + required: [predictionId, source, status, snapshotId, model] + properties: + predictionId: {$ref: '#/components/schemas/PredictionId'} + source: + $ref: '#/components/schemas/PredictionSource' + name: {type: string, minLength: 1, maxLength: 128} + status: + $ref: '#/components/schemas/PredictionStatus' + snapshotId: {type: integer, format: int64, minimum: 1} + model: + $ref: '#/components/schemas/Identifier' + output: + $ref: '#/components/schemas/PredictionOutput' + startTime: + type: integer + format: int64 + minimum: 0 + description: Start time as Unix epoch milliseconds in UTC. + endTime: + type: integer + format: int64 + minimum: 0 + description: End time as Unix epoch milliseconds in UTC. + ListPredictionsResponse: + type: object + required: [predictions] + properties: + predictions: + type: array + items: + $ref: '#/components/schemas/GetPredictionResponse' + nextPageToken: + type: [ string, "null" ] ConsumerInfo: type: object properties: diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java new file mode 100644 index 000000000000..5601a02414aa --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/Prediction.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.catalog; + +import org.apache.paimon.annotation.Experimental; + +import javax.annotation.Nullable; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Durable metadata for one asynchronous prediction job. */ +@Experimental +public class Prediction { + + private final String predictionId; + private final PredictionSource source; + private final @Nullable String name; + private final PredictionStatus status; + private final long snapshotId; + private final Identifier model; + private final @Nullable PredictionOutput output; + private final @Nullable Long startTime; + private final @Nullable Long endTime; + + public Prediction( + String predictionId, + PredictionSource source, + @Nullable String name, + PredictionStatus status, + long snapshotId, + Identifier model, + @Nullable PredictionOutput output, + @Nullable Long startTime, + @Nullable Long endTime) { + checkNonBlank(predictionId, "predictionId"); + checkArgument(source != null, "source must not be null"); + checkArgument(status != null, "status must not be null"); + checkArgument(snapshotId > 0, "snapshotId must be positive"); + checkArgument(model != null, "model must not be null"); + checkOptionalNonNegative(startTime, "startTime"); + checkOptionalNonNegative(endTime, "endTime"); + this.predictionId = predictionId; + this.source = source; + this.name = name; + this.status = status; + this.snapshotId = snapshotId; + this.model = model; + this.output = output; + this.startTime = startTime; + this.endTime = endTime; + } + + public String predictionId() { + return predictionId; + } + + public PredictionSource source() { + return source; + } + + @Nullable + public String name() { + return name; + } + + public PredictionStatus status() { + return status; + } + + public long snapshotId() { + return snapshotId; + } + + public Identifier model() { + return model; + } + + @Nullable + public PredictionOutput output() { + return output; + } + + /** Start time as Unix epoch milliseconds in UTC, or null if not started. */ + @Nullable + public Long startTime() { + return startTime; + } + + /** End time as Unix epoch milliseconds in UTC, or null if not terminated. */ + @Nullable + public Long endTime() { + return endTime; + } + + private static void checkNonBlank(String value, String name) { + checkArgument(value != null && !value.trim().isEmpty(), "%s must not be blank", name); + } + + private static void checkOptionalNonNegative(@Nullable Long value, String name) { + checkArgument(value == null || value >= 0, "%s must not be negative", name); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionOutput.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionOutput.java new file mode 100644 index 000000000000..2ea3d165036e --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionOutput.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.catalog; + +import org.apache.paimon.annotation.Experimental; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Output of an asynchronous prediction. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class PredictionOutput { + + private final PredictionSource table; + + @JsonCreator + public PredictionOutput(@JsonProperty("table") PredictionSource table) { + checkArgument(table != null, "table must not be null"); + this.table = table; + } + + @JsonGetter("table") + public PredictionSource table() { + return table; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSource.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSource.java new file mode 100644 index 000000000000..4b3eeeea5e50 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSource.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.catalog; + +import org.apache.paimon.annotation.Experimental; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Source table of a prediction job. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class PredictionSource { + + private final String database; + private final String table; + + @JsonCreator + public PredictionSource( + @JsonProperty("database") String database, @JsonProperty("table") String table) { + checkNonBlank(database, "database"); + checkNonBlank(table, "table"); + this.database = database; + this.table = table; + } + + public PredictionSource(Identifier identifier) { + this(identifier.getDatabaseName(), identifier.getObjectName()); + } + + @JsonGetter("database") + public String database() { + return database; + } + + @JsonGetter("table") + public String table() { + return table; + } + + public Identifier toIdentifier() { + return Identifier.create(database, table); + } + + private static void checkNonBlank(String value, String name) { + checkArgument(value != null && !value.trim().isEmpty(), "%s must not be blank", name); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java new file mode 100644 index 000000000000..d0f98313f288 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionSpec.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.catalog; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.annotation.Experimental; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** + * Provider-neutral specification for an asynchronous prediction over a table snapshot. + * + *

{@code options} uses Paimon dynamic table option names. Time travel uses {@code + * scan.snapshot-id} or {@code scan.tag-name}. + */ +@Experimental +public class PredictionSpec { + + private final @Nullable String name; + private final Identifier model; + private final Map inputs; + private final List mergeKeyColumns; + private final String instructions; + private final @Nullable String where; + private final Map options; + private final @Nullable String idempotencyToken; + + public PredictionSpec( + @Nullable String name, + Identifier model, + Map inputs, + @Nullable List mergeKeyColumns, + String instructions, + @Nullable String where, + @Nullable Map options, + @Nullable String idempotencyToken) { + checkOptionalNonBlank(name, "name"); + checkArgument(model != null, "model must not be null"); + checkNonBlank(model.getDatabaseName(), "model database"); + checkNonBlank(model.getObjectName(), "model object"); + checkNonBlank(instructions, "instructions"); + checkOptionalNonBlank(where, "where"); + checkOptionalNonBlank(idempotencyToken, "idempotencyToken"); + checkArgument( + idempotencyToken == null || idempotencyToken.length() <= 64, + "idempotencyToken must not exceed 64 characters"); + this.name = name; + this.model = model; + this.inputs = copyStringMap(inputs, "inputs", true); + this.mergeKeyColumns = copyOptionalStringList(mergeKeyColumns, "mergeKeyColumns"); + this.instructions = instructions; + this.where = where; + this.idempotencyToken = idempotencyToken; + this.options = + options == null ? Collections.emptyMap() : copyStringMap(options, "options", false); + CoreOptions coreOptions = new CoreOptions(this.options); + Long snapshotId = coreOptions.scanSnapshotId(); + String tagName = coreOptions.scanTagName(); + checkArgument(snapshotId == null || snapshotId > 0, "scan.snapshot-id must be positive"); + checkArgument( + snapshotId == null || tagName == null, + "scan.snapshot-id and scan.tag-name are mutually exclusive"); + } + + @Nullable + public String name() { + return name; + } + + public Identifier model() { + return model; + } + + public Map inputs() { + return inputs; + } + + public List mergeKeyColumns() { + return mergeKeyColumns; + } + + public String instructions() { + return instructions; + } + + @Nullable + public String where() { + return where; + } + + public Map options() { + return options; + } + + @Nullable + public String idempotencyToken() { + return idempotencyToken; + } + + private static Map copyStringMap( + Map values, String name, boolean nonEmpty) { + checkArgument(values != null, "%s must not be null", name); + checkArgument(!nonEmpty || !values.isEmpty(), "%s must not be empty", name); + Map copy = new LinkedHashMap<>(); + values.forEach( + (key, value) -> { + checkNonBlank(key, name + " key"); + checkNonBlank(value, name + " value"); + copy.put(key, value); + }); + return Collections.unmodifiableMap(copy); + } + + private static List copyStringList(List values, String name) { + checkArgument(values != null, "%s must not be null", name); + List copy = new ArrayList<>(); + for (String value : values) { + checkNonBlank(value, name + " value"); + checkArgument(!copy.contains(value), "%s must not contain duplicates", name); + copy.add(value); + } + return Collections.unmodifiableList(copy); + } + + private static List copyOptionalStringList(@Nullable List values, String name) { + return values == null ? Collections.emptyList() : copyStringList(values, name); + } + + private static void checkOptionalNonBlank(@Nullable String value, String name) { + if (value != null) { + checkNonBlank(value, name); + } + } + + private static void checkNonBlank(String value, String name) { + checkArgument(value != null && !value.trim().isEmpty(), "%s must not be blank", name); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionStatus.java b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionStatus.java new file mode 100644 index 000000000000..527004d191ba --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/catalog/PredictionStatus.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.catalog; + +import org.apache.paimon.annotation.Experimental; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Lifecycle and optional termination details of an asynchronous prediction. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class PredictionStatus { + + /** Current lifecycle state. */ + public enum State { + PENDING, + RUNNING, + TERMINATING, + TERMINATED + } + + /** Result of a terminated prediction. */ + public enum TerminationCode { + SUCCESS, + SUCCESS_WITH_FAILURES, + FAILED, + CANCELED + } + + /** Result and diagnostic message available after termination. */ + @JsonIgnoreProperties(ignoreUnknown = true) + public static class TerminationDetails { + + private final TerminationCode code; + private final @Nullable String message; + + @JsonCreator + public TerminationDetails( + @JsonProperty("code") TerminationCode code, + @Nullable @JsonProperty("message") String message) { + checkArgument(code != null, "code must not be null"); + this.code = code; + this.message = message; + } + + @JsonGetter("code") + public TerminationCode code() { + return code; + } + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter("message") + public String message() { + return message; + } + } + + private final State state; + private final @Nullable TerminationDetails terminationDetails; + + @JsonCreator + public PredictionStatus( + @JsonProperty("state") State state, + @Nullable @JsonProperty("terminationDetails") TerminationDetails terminationDetails) { + checkArgument(state != null, "state must not be null"); + checkArgument( + state != State.TERMINATED || terminationDetails != null, + "terminationDetails must be present for TERMINATED state"); + checkArgument( + state == State.TERMINATING + || state == State.TERMINATED + || terminationDetails == null, + "terminationDetails must not be present before termination starts"); + this.state = state; + this.terminationDetails = terminationDetails; + } + + @JsonGetter("state") + public State state() { + return state; + } + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter("terminationDetails") + public TerminationDetails terminationDetails() { + return terminationDetails; + } + + public boolean isTerminal() { + return state == State.TERMINATED; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index d017d1544848..7f1ebfbd7a5a 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -24,6 +24,7 @@ import org.apache.paimon.annotation.Public; import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.PredictionStatus; import org.apache.paimon.consumer.ConsumerInfo; import org.apache.paimon.function.FunctionChange; import org.apache.paimon.management.DataPolicy; @@ -72,6 +73,7 @@ import org.apache.paimon.rest.requests.RevokePermissionRequest; import org.apache.paimon.rest.requests.RollbackSchemaRequest; import org.apache.paimon.rest.requests.RollbackTableRequest; +import org.apache.paimon.rest.requests.SubmitPredictionRequest; import org.apache.paimon.rest.requests.UpsertLabelRequest; import org.apache.paimon.rest.requests.UpsertSemanticViewRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; @@ -86,6 +88,7 @@ import org.apache.paimon.rest.responses.GetFunctionResponse; import org.apache.paimon.rest.responses.GetLabelResponse; import org.apache.paimon.rest.responses.GetModelResponse; +import org.apache.paimon.rest.responses.GetPredictionResponse; import org.apache.paimon.rest.responses.GetSchemaResponse; import org.apache.paimon.rest.responses.GetSemanticViewResponse; import org.apache.paimon.rest.responses.GetTableResponse; @@ -105,6 +108,7 @@ import org.apache.paimon.rest.responses.ListPartitionsResponse; import org.apache.paimon.rest.responses.ListPermissionsResponse; import org.apache.paimon.rest.responses.ListPoliciesResponse; +import org.apache.paimon.rest.responses.ListPredictionsResponse; import org.apache.paimon.rest.responses.ListSchemasResponse; import org.apache.paimon.rest.responses.ListSemanticViewsResponse; import org.apache.paimon.rest.responses.ListSnapshotsResponse; @@ -204,6 +208,8 @@ public class RESTApi { public static final String TAG_NAME_PREFIX = "tagNamePrefix"; public static final String ENTITY_TYPE = "entityType"; public static final String ENTITY_NAME = "entityName"; + public static final String PREDICTION_NAME = "name"; + public static final String PREDICTION_STATE = "state"; public static final long TOKEN_EXPIRATION_SAFE_TIME_MILLIS = 3_600_000L; @@ -1812,6 +1818,56 @@ public void dropModel(Identifier identifier) { restAuthFunction); } + /** Submit an asynchronous prediction over a table. */ + public GetPredictionResponse submitPrediction(SubmitPredictionRequest request) { + return client.post( + resourcePaths.predictions(), + request, + GetPredictionResponse.class, + restAuthFunction); + } + + /** Get one asynchronous prediction by its unique id. */ + public GetPredictionResponse getPrediction(String predictionId) { + return client.get( + resourcePaths.prediction(predictionId), + GetPredictionResponse.class, + restAuthFunction); + } + + /** List asynchronous predictions, ordered newest first by the server. */ + public PagedList listPredictions( + @Nullable Identifier source, + @Nullable String name, + @Nullable PredictionStatus.State state, + @Nullable Integer maxResults, + @Nullable String pageToken) { + Map queryParams = + buildPagedQueryParams( + maxResults, + pageToken, + Pair.of("database", source == null ? null : source.getDatabaseName()), + Pair.of("table", source == null ? null : source.getObjectName()), + Pair.of(PREDICTION_NAME, name), + Pair.of(PREDICTION_STATE, state == null ? null : state.name())); + ListPredictionsResponse response = + client.get( + resourcePaths.predictions(), + queryParams, + ListPredictionsResponse.class, + restAuthFunction); + return new PagedList<>(response.getPredictions(), response.getNextPageToken()); + } + + /** Request cancellation and return the updated prediction metadata. */ + public GetPredictionResponse cancelPrediction(String predictionId) { + return client.post( + resourcePaths.cancelPrediction(predictionId), + null, + GetPredictionResponse.class, + restAuthFunction); + } + /** * Alter a function. * diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java index dfbf2b489a31..8858524ca18b 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -47,6 +47,7 @@ public class ResourcePaths { protected static final String REGISTER = "register"; protected static final String FUNCTIONS = "functions"; protected static final String MODELS = "models"; + protected static final String PREDICTIONS = "predictions"; protected static final String FUNCTION_DETAILS = "function-details"; protected static final String PERMISSIONS = "permissions"; protected static final String POLICIES = "policies"; @@ -151,16 +152,29 @@ public String database(String databaseName) { } public String models(String databaseName) { - checkModelPathSegment(databaseName, "databaseName"); + checkPathSegment(databaseName, "databaseName"); return SLASH.join(V1, prefix, DATABASES, encodeString(databaseName), MODELS); } public String model(String databaseName, String modelName) { - checkModelPathSegment(modelName, "modelName"); + checkPathSegment(modelName, "modelName"); return SLASH.join(models(databaseName), encodeString(modelName)); } - private static void checkModelPathSegment(String value, String name) { + public String predictions() { + return SLASH.join(V1, prefix, PREDICTIONS); + } + + public String prediction(String predictionId) { + checkPathSegment(predictionId, "predictionId"); + return SLASH.join(predictions(), encodeString(predictionId)); + } + + public String cancelPrediction(String predictionId) { + return SLASH.join(prediction(predictionId), "cancel"); + } + + private static void checkPathSegment(String value, String name) { checkArgument(value != null && !value.trim().isEmpty(), "%s must not be blank", name); checkArgument( !".".equals(value) && !"..".equals(value), "%s must not be a dot segment", name); diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java new file mode 100644 index 000000000000..737de6492a71 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/SubmitPredictionRequest.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.PredictionSource; +import org.apache.paimon.catalog.PredictionSpec; +import org.apache.paimon.rest.RESTRequest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Request for submitting an asynchronous prediction job. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class SubmitPredictionRequest implements RESTRequest { + + private static final String FIELD_NAME = "name"; + private static final String FIELD_SOURCE = "source"; + private static final String FIELD_IDEMPOTENCY_TOKEN = "idempotencyToken"; + private static final String FIELD_MODEL = "model"; + private static final String FIELD_INPUTS = "inputs"; + private static final String FIELD_MERGE_KEY_COLUMNS = "mergeKeyColumns"; + private static final String FIELD_INSTRUCTIONS = "instructions"; + private static final String FIELD_WHERE = "where"; + private static final String FIELD_OPTIONS = "options"; + + private final String idempotencyToken; + private final PredictionSource source; + private final PredictionSpec spec; + + @JsonCreator + public SubmitPredictionRequest( + @JsonProperty(FIELD_SOURCE) PredictionSource source, + @Nullable @JsonProperty(FIELD_NAME) String name, + @Nullable @JsonProperty(FIELD_IDEMPOTENCY_TOKEN) String idempotencyToken, + @JsonProperty(FIELD_MODEL) Identifier model, + @JsonProperty(FIELD_INPUTS) Map inputs, + @Nullable @JsonProperty(FIELD_MERGE_KEY_COLUMNS) List mergeKeyColumns, + @JsonProperty(FIELD_INSTRUCTIONS) String instructions, + @Nullable @JsonProperty(FIELD_WHERE) String where, + @Nullable @JsonProperty(FIELD_OPTIONS) Map options) { + checkArgument(source != null, "source must not be null"); + checkArgument(model != null, "model must not be null"); + checkArgument( + idempotencyToken == null || !idempotencyToken.trim().isEmpty(), + "idempotencyToken must not be blank"); + checkArgument( + idempotencyToken == null || idempotencyToken.length() <= 64, + "idempotencyToken must not exceed 64 characters"); + this.source = source; + this.idempotencyToken = + idempotencyToken == null ? UUID.randomUUID().toString() : idempotencyToken; + this.spec = + new PredictionSpec( + name, + model, + inputs, + mergeKeyColumns, + instructions, + where, + options, + this.idempotencyToken); + } + + public SubmitPredictionRequest(Identifier source, PredictionSpec spec) { + this( + new PredictionSource(source), + spec.name(), + spec.idempotencyToken(), + spec.model(), + spec.inputs(), + spec.mergeKeyColumns(), + spec.instructions(), + spec.where(), + spec.options()); + } + + @JsonGetter(FIELD_SOURCE) + public PredictionSource getSource() { + return source; + } + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter(FIELD_NAME) + public String getName() { + return spec.name(); + } + + @JsonGetter(FIELD_IDEMPOTENCY_TOKEN) + public String getIdempotencyToken() { + return idempotencyToken; + } + + @JsonGetter(FIELD_MODEL) + public Identifier getModel() { + return spec.model(); + } + + @JsonGetter(FIELD_INPUTS) + public Map getInputs() { + return spec.inputs(); + } + + @JsonInclude(JsonInclude.Include.NON_EMPTY) + @JsonGetter(FIELD_MERGE_KEY_COLUMNS) + public List getMergeKeyColumns() { + return spec.mergeKeyColumns(); + } + + @JsonGetter(FIELD_INSTRUCTIONS) + public String getInstructions() { + return spec.instructions(); + } + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter(FIELD_WHERE) + public String getWhere() { + return spec.where(); + } + + @JsonGetter(FIELD_OPTIONS) + public Map getOptions() { + return spec.options(); + } + + public PredictionSpec toPredictionSpec() { + return spec; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java index b5a6f1c792a7..ff47500d40f8 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java @@ -57,6 +57,7 @@ public class ErrorResponse implements RESTResponse { public static final String RESOURCE_TYPE_FUNCTION = "FUNCTION"; public static final String RESOURCE_TYPE_MODEL = "MODEL"; + public static final String RESOURCE_TYPE_PREDICTION = "PREDICTION"; public static final String RESOURCE_TYPE_POLICY = "POLICY"; diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java new file mode 100644 index 000000000000..c91054463443 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/GetPredictionResponse.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.Prediction; +import org.apache.paimon.catalog.PredictionOutput; +import org.apache.paimon.catalog.PredictionSource; +import org.apache.paimon.catalog.PredictionStatus; +import org.apache.paimon.rest.RESTResponse; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +/** Response containing durable metadata for one asynchronous prediction job. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class GetPredictionResponse implements RESTResponse { + + private final String predictionId; + private final PredictionSource source; + private final @Nullable String name; + private final PredictionStatus status; + private final long snapshotId; + private final Identifier model; + private final @Nullable PredictionOutput output; + private final @Nullable Long startTime; + private final @Nullable Long endTime; + + @JsonCreator + public GetPredictionResponse( + @JsonProperty("predictionId") String predictionId, + @JsonProperty("source") PredictionSource source, + @Nullable @JsonProperty("name") String name, + @JsonProperty("status") PredictionStatus status, + @JsonProperty("snapshotId") long snapshotId, + @JsonProperty("model") Identifier model, + @Nullable @JsonProperty("output") PredictionOutput output, + @Nullable @JsonProperty("startTime") Long startTime, + @Nullable @JsonProperty("endTime") Long endTime) { + this.predictionId = predictionId; + this.source = source; + this.name = name; + this.status = status; + this.snapshotId = snapshotId; + this.model = model; + this.output = output; + this.startTime = startTime; + this.endTime = endTime; + } + + public GetPredictionResponse(Prediction prediction) { + this( + prediction.predictionId(), + prediction.source(), + prediction.name(), + prediction.status(), + prediction.snapshotId(), + prediction.model(), + prediction.output(), + prediction.startTime(), + prediction.endTime()); + } + + @JsonGetter("predictionId") + public String getPredictionId() { + return predictionId; + } + + @JsonGetter("source") + public PredictionSource getSource() { + return source; + } + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter("name") + public String getName() { + return name; + } + + @JsonGetter("status") + public PredictionStatus getStatus() { + return status; + } + + @JsonGetter("snapshotId") + public long getSnapshotId() { + return snapshotId; + } + + @JsonGetter("model") + public Identifier getModel() { + return model; + } + + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter("output") + public PredictionOutput getOutput() { + return output; + } + + /** Start time as Unix epoch milliseconds in UTC, or null if not started. */ + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter("startTime") + public Long getStartTime() { + return startTime; + } + + /** End time as Unix epoch milliseconds in UTC, or null if not terminated. */ + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter("endTime") + public Long getEndTime() { + return endTime; + } + + public Prediction toPrediction() { + return new Prediction( + predictionId, source, name, status, snapshotId, model, output, startTime, endTime); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListPredictionsResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListPredictionsResponse.java new file mode 100644 index 000000000000..9a78780f988f --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListPredictionsResponse.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import java.util.Collections; +import java.util.List; + +/** Response for listing asynchronous prediction jobs. */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class ListPredictionsResponse implements PagedResponse { + + private final List predictions; + private final @Nullable String nextPageToken; + + @JsonCreator + public ListPredictionsResponse( + @Nullable @JsonProperty("predictions") List predictions, + @Nullable @JsonProperty("nextPageToken") String nextPageToken) { + this.predictions = predictions == null ? Collections.emptyList() : predictions; + this.nextPageToken = nextPageToken; + } + + @JsonGetter("predictions") + public List getPredictions() { + return predictions; + } + + @Override + @Nullable + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonGetter("nextPageToken") + public String getNextPageToken() { + return nextPageToken; + } + + @Override + public List data() { + return predictions; + } +} diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiCoverageTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiCoverageTest.java index 32da66f2abc2..e7c4d9cc9813 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiCoverageTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiCoverageTest.java @@ -79,7 +79,14 @@ class RESTApiCoverageTest { /** RESTApi methods with no entry in DLFOpenApiActions; they are signed without an action. */ - private static final Set WITHOUT_ACTION = Collections.emptySet(); + private static final Set WITHOUT_ACTION = + Collections.unmodifiableSet( + new TreeSet<>( + Arrays.asList( + "cancelPrediction", + "getPrediction", + "listPredictions", + "submitPrediction"))); /** A value per parameter type; "SELECT" is valid both as a name and as an access. */ private static final Map, Object> ARGUMENTS = arguments(); diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java new file mode 100644 index 000000000000..9d65c53d1789 --- /dev/null +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTApiPredictionTest.java @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.Prediction; +import org.apache.paimon.catalog.PredictionOutput; +import org.apache.paimon.catalog.PredictionSource; +import org.apache.paimon.catalog.PredictionSpec; +import org.apache.paimon.catalog.PredictionStatus; +import org.apache.paimon.rest.requests.SubmitPredictionRequest; +import org.apache.paimon.rest.responses.GetPredictionResponse; +import org.apache.paimon.rest.responses.ListPredictionsResponse; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Wire-format tests for asynchronous prediction metadata. */ +class RESTApiPredictionTest { + + @Test + void testRequestAndResponseWireFormat() throws Exception { + SubmitPredictionRequest request = + new SubmitPredictionRequest(Identifier.create("quality", "clips"), spec()); + Map requestMap = RESTApi.fromJson(RESTApi.toJson(request), Map.class); + assertThat(requestMap.get("idempotencyToken")).isInstanceOf(String.class); + assertThat((String) requestMap.get("idempotencyToken")).isNotBlank(); + assertThat(RESTApi.fromJson(RESTApi.toJson(request), Map.class).get("idempotencyToken")) + .isEqualTo(requestMap.get("idempotencyToken")); + assertThat(request.toPredictionSpec().idempotencyToken()) + .isEqualTo(request.getIdempotencyToken()); + assertThat(requestMap) + .containsEntry("instructions", "Classify") + .doesNotContainKeys( + "name", + "prompt", + "outputSchema", + "where", + "snapshotId", + "tagName", + "resultColumns"); + assertThat(requestMap.get("mergeKeyColumns")) + .isEqualTo(Collections.singletonList("clip_id")); + assertThat((Map) requestMap.get("source")) + .containsEntry("database", "quality") + .containsEntry("table", "clips"); + assertThat((Map) requestMap.get("model")) + .containsEntry("database", "quality") + .containsEntry("object", "qwen_vl"); + assertThat((Map) requestMap.get("options")) + .containsEntry("scan.snapshot-id", "7"); + + GetPredictionResponse response = new GetPredictionResponse(prediction()); + GetPredictionResponse decoded = + RESTApi.fromJson(RESTApi.toJson(response), GetPredictionResponse.class); + assertThat(decoded.toPrediction().status().state()) + .isEqualTo(PredictionStatus.State.PENDING); + assertThat(decoded.toPrediction().status().terminationDetails()).isNull(); + assertThat(decoded.getName()).isNull(); + assertThat(RESTApi.fromJson(RESTApi.toJson(response), Map.class)) + .doesNotContainKeys( + "createdAt", + "startedAt", + "finishedAt", + "modelVersion", + "outputTable", + "output", + "startTime", + "endTime"); + + GetPredictionResponse completed = + new GetPredictionResponse( + new Prediction( + "prediction-2", + new PredictionSource("quality", "clips"), + null, + new PredictionStatus( + PredictionStatus.State.TERMINATED, + new PredictionStatus.TerminationDetails( + PredictionStatus.TerminationCode.SUCCESS, null)), + 7, + Identifier.create("quality", "qwen_vl"), + new PredictionOutput( + new PredictionSource("system", "prediction_results")), + 10L, + 20L)); + Map completedMap = RESTApi.fromJson(RESTApi.toJson(completed), Map.class); + assertThat(completedMap) + .containsEntry("startTime", 10) + .containsEntry("endTime", 20) + .doesNotContainKeys( + "createdAt", "startedAt", "finishedAt", "modelVersion", "outputTable"); + Map output = (Map) completedMap.get("output"); + assertThat((Map) output.get("table")) + .containsEntry("database", "system") + .containsEntry("table", "prediction_results"); + assertThatThrownBy(() -> new PredictionOutput(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("table"); + + ListPredictionsResponse empty = RESTApi.fromJson("{}", ListPredictionsResponse.class); + assertThat(empty.getPredictions()).isEmpty(); + assertThat(RESTApi.fromJson(RESTApi.toJson(empty), Map.class)) + .containsOnlyKeys("predictions") + .containsEntry("predictions", Collections.emptyList()); + } + + @Test + void testStructuredModelPreservesDots() throws Exception { + PredictionSpec spec = + new PredictionSpec( + null, + Identifier.create("quality.archive", "qwen.vl"), + Collections.singletonMap("video", "content"), + null, + "Classify", + null, + Collections.emptyMap(), + "retry-1"); + SubmitPredictionRequest request = + new SubmitPredictionRequest(Identifier.create("quality", "clips"), spec); + SubmitPredictionRequest decoded = + RESTApi.fromJson(RESTApi.toJson(request), SubmitPredictionRequest.class); + + assertThat(decoded.getModel()).isEqualTo(Identifier.create("quality.archive", "qwen.vl")); + assertThat(decoded.getIdempotencyToken()).isEqualTo("retry-1"); + } + + @Test + void testSpecValidation() { + assertThatThrownBy( + () -> + new PredictionSpec( + null, + Identifier.create("quality", "qwen_vl"), + Collections.emptyMap(), + null, + "Classify", + null, + Collections.emptyMap(), + null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("inputs must not be empty"); + + assertThatThrownBy( + () -> + new PredictionSpec( + null, + Identifier.create("quality", "qwen_vl"), + Collections.singletonMap("video", "content"), + Arrays.asList("clip_id", "clip_id"), + "Classify", + null, + Collections.emptyMap(), + null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("mergeKeyColumns must not contain duplicates"); + + assertThatThrownBy( + () -> + new PredictionSpec( + null, + Identifier.create("quality", "qwen_vl"), + Collections.singletonMap("video", "content"), + null, + "Classify", + null, + Collections.singletonMap("scan.snapshot-id", "0"), + null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("scan.snapshot-id must be positive"); + + Map conflictingOptions = new LinkedHashMap<>(); + conflictingOptions.put("scan.snapshot-id", "7"); + conflictingOptions.put("scan.tag-name", "training"); + assertThatThrownBy( + () -> + new PredictionSpec( + null, + Identifier.create("quality", "qwen_vl"), + Collections.singletonMap("video", "content"), + null, + "Classify", + null, + conflictingOptions, + null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("mutually exclusive"); + + assertThatThrownBy( + () -> + new PredictionSpec( + null, + Identifier.create("", "qwen_vl"), + Collections.singletonMap("video", "content"), + null, + "Classify", + null, + Collections.emptyMap(), + null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("model database"); + + assertThatThrownBy( + () -> + new PredictionSpec( + null, + Identifier.create("quality", "qwen_vl"), + Collections.singletonMap("video", "content"), + null, + "Classify", + null, + Collections.emptyMap(), + String.join("", Collections.nCopies(65, "x")))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("64 characters"); + } + + @Test + void testTerminalStates() { + assertThat(new PredictionStatus(PredictionStatus.State.PENDING, null).isTerminal()) + .isFalse(); + assertThat(new PredictionStatus(PredictionStatus.State.RUNNING, null).isTerminal()) + .isFalse(); + assertThat(new PredictionStatus(PredictionStatus.State.TERMINATING, null).isTerminal()) + .isFalse(); + for (PredictionStatus.TerminationCode code : + Arrays.asList( + PredictionStatus.TerminationCode.CANCELED, + PredictionStatus.TerminationCode.SUCCESS)) { + PredictionStatus terminated = + new PredictionStatus( + PredictionStatus.State.TERMINATED, + new PredictionStatus.TerminationDetails(code, null)); + assertThat(terminated.isTerminal()).isTrue(); + assertThat(terminated.terminationDetails().code()).isEqualTo(code); + } + assertThatThrownBy(() -> new PredictionStatus(PredictionStatus.State.TERMINATED, null)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy( + () -> + new PredictionStatus( + PredictionStatus.State.RUNNING, + new PredictionStatus.TerminationDetails( + PredictionStatus.TerminationCode.SUCCESS, null))) + .isInstanceOf(IllegalArgumentException.class); + } + + private static PredictionSpec spec() { + Map inputs = new LinkedHashMap<>(); + inputs.put("video", "content"); + return new PredictionSpec( + null, + Identifier.create("quality", "qwen_vl"), + inputs, + Collections.singletonList("clip_id"), + "Classify", + null, + Collections.singletonMap("scan.snapshot-id", "7"), + null); + } + + private static Prediction prediction() { + return new Prediction( + "prediction-1", + new PredictionSource("quality", "clips"), + null, + new PredictionStatus(PredictionStatus.State.PENDING, null), + 7, + Identifier.create("quality", "qwen_vl"), + null, + null, + null); + } +} diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiActionsTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiActionsTest.java index 9e4a4f1c36b3..d456239417fe 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiActionsTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/auth/DLFOpenApiActionsTest.java @@ -26,6 +26,7 @@ import org.junit.jupiter.api.Test; import java.io.IOException; +import java.util.Arrays; import java.util.HashSet; import java.util.Set; import java.util.TreeSet; @@ -38,6 +39,15 @@ /** Test for {@link DLFOpenApiActions}. */ public class DLFOpenApiActionsTest { + private static final Set REST_ONLY_OPERATIONS = + new HashSet<>( + Arrays.asList( + OpenApiSpec.normalize("POST /v1/{prefix}/predictions"), + OpenApiSpec.normalize("GET /v1/{prefix}/predictions"), + OpenApiSpec.normalize("GET /v1/{prefix}/predictions/{predictionId}"), + OpenApiSpec.normalize( + "POST /v1/{prefix}/predictions/{predictionId}/cancel"))); + private final ResourcePaths paths = new ResourcePaths("clg-paimon-1"); /** Each template, filled in, must come back to its own action rather than an earlier one. */ @@ -59,12 +69,13 @@ public void testOperationsMatchTheOpenApiSpec() throws IOException { for (String operation : OpenApiSpec.operations()) { spec.add(OpenApiSpec.normalize(operation)); } + spec.removeAll(REST_ONLY_OPERATIONS); Set registered = new TreeSet<>(); for (String[] operation : DLFOpenApiActions.operations()) { registered.add(OpenApiSpec.normalize(operation[0] + " " + operation[1])); } assertThat(registered) - .as("DLFOpenApiActions must register exactly the spec operations") + .as("DLFOpenApiActions must register the POP-enabled spec operations") .containsExactlyInAnyOrderElementsOf(spec); } diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java index 3c2c64357630..248c6d75c115 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java @@ -225,7 +225,8 @@ public class RequestJacksonCompatibilityTest { RenameTableRequest.class, ReplaceTableRequest.class, RevokePermissionRequest.class, - RollbackTableRequest.class) + RollbackTableRequest.class, + SubmitPredictionRequest.class) .collect(Collectors.toSet()); @ParameterizedTest(name = "{0}") diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java index 0aa2795bad9e..0eae9bb8f99a 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/Catalog.java @@ -1197,6 +1197,34 @@ default void dropModel(Identifier identifier, boolean ignoreIfNotExists) throw new UnsupportedOperationException("Current Catalog does not support models"); } + // ======================= Prediction methods =============================== + + /** Submit an asynchronous prediction over a table and its fixed snapshot. */ + default Prediction submitPrediction(Identifier identifier, PredictionSpec spec) + throws DatabaseNotExistException, TableNotExistException, ModelNotExistException { + throw new UnsupportedOperationException("Current Catalog does not support predictions"); + } + + /** Get one asynchronous prediction by its unique id. */ + default Prediction getPrediction(String predictionId) throws PredictionNotExistException { + throw new UnsupportedOperationException("Current Catalog does not support predictions"); + } + + /** List asynchronous predictions, optionally filtered by source table. */ + default PagedList listPredictions( + @Nullable Identifier source, + @Nullable String name, + @Nullable PredictionStatus.State state, + @Nullable Integer maxResults, + @Nullable String pageToken) { + throw new UnsupportedOperationException("Current Catalog does not support predictions"); + } + + /** Request cancellation and return the updated prediction metadata. */ + default Prediction cancelPrediction(String predictionId) throws PredictionNotExistException { + throw new UnsupportedOperationException("Current Catalog does not support predictions"); + } + // ======================= Function methods =============================== /** @@ -1937,6 +1965,27 @@ public Identifier identifier() { } } + /** Exception for trying to access a prediction that doesn't exist. */ + class PredictionNotExistException extends Exception { + + private static final String MSG = "Prediction %s doesn't exist."; + + private final String predictionId; + + public PredictionNotExistException(String predictionId) { + this(predictionId, null); + } + + public PredictionNotExistException(String predictionId, @Nullable Throwable cause) { + super(String.format(MSG, predictionId), cause); + this.predictionId = predictionId; + } + + public String predictionId() { + return predictionId; + } + } + /** Exception for trying to create a function that already exists. */ class FunctionAlreadyExistException extends Exception { diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java index 48383b090066..8b72c9496767 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/DelegateCatalog.java @@ -407,6 +407,32 @@ public void dropModel(Identifier identifier, boolean ignoreIfNotExists) wrapped.dropModel(identifier, ignoreIfNotExists); } + @Override + public Prediction submitPrediction(Identifier identifier, PredictionSpec spec) + throws DatabaseNotExistException, TableNotExistException, ModelNotExistException { + return wrapped.submitPrediction(identifier, spec); + } + + @Override + public Prediction getPrediction(String predictionId) throws PredictionNotExistException { + return wrapped.getPrediction(predictionId); + } + + @Override + public PagedList listPredictions( + @Nullable Identifier source, + @Nullable String name, + @Nullable PredictionStatus.State state, + @Nullable Integer maxResults, + @Nullable String pageToken) { + return wrapped.listPredictions(source, name, state, maxResults, pageToken); + } + + @Override + public Prediction cancelPrediction(String predictionId) throws PredictionNotExistException { + return wrapped.cancelPrediction(predictionId); + } + @Override public List listFunctions(String databaseName) throws DatabaseNotExistException { return wrapped.listFunctions(databaseName); diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index fd5a0df13732..d06405196110 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -31,6 +31,9 @@ import org.apache.paimon.catalog.Database; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.catalog.Model; +import org.apache.paimon.catalog.Prediction; +import org.apache.paimon.catalog.PredictionSpec; +import org.apache.paimon.catalog.PredictionStatus; import org.apache.paimon.catalog.PropertyChange; import org.apache.paimon.catalog.TableMetadata; import org.apache.paimon.catalog.TableQueryAuthResult; @@ -58,6 +61,7 @@ import org.apache.paimon.rest.exceptions.NotImplementedException; import org.apache.paimon.rest.exceptions.ServiceFailureException; import org.apache.paimon.rest.requests.CreateModelRequest; +import org.apache.paimon.rest.requests.SubmitPredictionRequest; import org.apache.paimon.rest.responses.AuthTableQueryResponse; import org.apache.paimon.rest.responses.ErrorResponse; import org.apache.paimon.rest.responses.GetDatabaseResponse; @@ -1210,6 +1214,84 @@ public void dropModel(Identifier identifier, boolean ignoreIfNotExists) } } + @Override + public Prediction submitPrediction(Identifier identifier, PredictionSpec spec) + throws DatabaseNotExistException, TableNotExistException, ModelNotExistException { + checkNotBranch(identifier, "submitPrediction"); + checkNotSystemTable(identifier, "submitPrediction"); + try { + return api.submitPrediction(new SubmitPredictionRequest(identifier, spec)) + .toPrediction(); + } catch (NoSuchResourceException e) { + if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_DATABASE)) { + String databaseName = + StringUtils.isNullOrWhitespaceOnly(e.resourceName()) + ? identifier.getDatabaseName() + : e.resourceName(); + throw new DatabaseNotExistException(databaseName, e); + } + if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_MODEL)) { + throw new ModelNotExistException(spec.model(), e); + } + if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_TABLE)) { + throw new TableNotExistException(identifier, e); + } + throw e; + } catch (NotImplementedException e) { + throw new UnsupportedOperationException("REST Catalog does not support predictions", e); + } + } + + @Override + public Prediction getPrediction(String predictionId) throws PredictionNotExistException { + try { + return api.getPrediction(predictionId).toPrediction(); + } catch (NoSuchResourceException e) { + if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_PREDICTION)) { + throw new PredictionNotExistException(predictionId, e); + } + throw e; + } catch (NotImplementedException e) { + throw new UnsupportedOperationException("REST Catalog does not support predictions", e); + } + } + + @Override + public PagedList listPredictions( + @Nullable Identifier source, + @Nullable String name, + @Nullable PredictionStatus.State state, + @Nullable Integer maxResults, + @Nullable String pageToken) { + try { + PagedList response = + api.listPredictions(source, name, state, maxResults, pageToken); + return new PagedList<>( + response.getElements().stream() + .map( + org.apache.paimon.rest.responses.GetPredictionResponse + ::toPrediction) + .collect(Collectors.toList()), + response.getNextPageToken()); + } catch (NotImplementedException e) { + throw new UnsupportedOperationException("REST Catalog does not support predictions", e); + } + } + + @Override + public Prediction cancelPrediction(String predictionId) throws PredictionNotExistException { + try { + return api.cancelPrediction(predictionId).toPrediction(); + } catch (NoSuchResourceException e) { + if (StringUtils.equals(e.resourceType(), ErrorResponse.RESOURCE_TYPE_PREDICTION)) { + throw new PredictionNotExistException(predictionId, e); + } + throw e; + } catch (NotImplementedException e) { + throw new UnsupportedOperationException("REST Catalog does not support predictions", e); + } + } + @Override public List listFunctions(String databaseName) throws DatabaseNotExistException { try { diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java index ecf653f58c0f..bdec50a83960 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java @@ -27,8 +27,13 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.catalog.Model; import org.apache.paimon.catalog.ModelField; +import org.apache.paimon.catalog.Prediction; +import org.apache.paimon.catalog.PredictionSpec; +import org.apache.paimon.catalog.PredictionStatus; import org.apache.paimon.catalog.ProviderModelDefinition; import org.apache.paimon.catalog.TableQueryAuthResult; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericRow; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.management.ListPermissionsRequest; @@ -72,6 +77,7 @@ import org.apache.paimon.table.sink.BatchTableCommit; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.RowKind; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.BuildVersions; import org.apache.paimon.utils.InstantiationUtil; @@ -91,6 +97,7 @@ import java.net.URI; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.AbstractMap; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -120,6 +127,7 @@ class MockRESTCatalogTest extends RESTCatalogTest { private RESTCatalogServer restCatalogServer; + private RESTCatalogPredictionTestControl predictionTestControl; private final String serverDefineHeaderName = "test-header"; private final String serverDefineHeaderValue = "test-value"; private String dataPath; @@ -190,6 +198,382 @@ void testModelCrudRoundTrip() throws Exception { .isInstanceOf(Catalog.DatabaseNotExistException.class); } + @Test + void testPredictionReferenceIntegration() throws Exception { + String database = "prediction_reference"; + Identifier sourceIdentifier = Identifier.create(database, "clips"); + Identifier modelIdentifier = Identifier.create(database, "echo_model"); + catalog.createDatabase(database, false); + catalog.createTable( + sourceIdentifier, + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("content", DataTypes.STRING()) + .build(), + false); + FileStoreTable source = (FileStoreTable) catalog.getTable(sourceIdentifier); + commitRows(source, GenericRow.of(1, BinaryString.fromString("first"))); + + ProviderModelDefinition definition = + new ProviderModelDefinition( + "reference", + "echo", + Collections.singletonList(new ModelField("text", "STRING", null)), + Collections.singletonList(new ModelField("response", "STRING", null)), + Collections.emptyMap()); + catalog.createModel( + modelIdentifier, + new Model(modelIdentifier, "Deterministic reference model", definition), + false); + + Runnable resumePredictionExecution = predictionTestControl.pauseExecution(); + Prediction submitted = + catalog.submitPrediction( + sourceIdentifier, + new PredictionSpec( + "snapshot-readback", + modelIdentifier, + Collections.singletonMap("text", "content"), + Collections.singletonList("id"), + "Echo the input", + null, + Collections.emptyMap(), + "reference-success")); + long pinnedSnapshot = submitted.snapshotId(); + + commitRows(source, GenericRow.of(2, BinaryString.fromString("second"))); + assertThat( + ((FileStoreTable) catalog.getTable(sourceIdentifier)) + .snapshotManager() + .latestSnapshotId()) + .isGreaterThan(pinnedSnapshot); + + resumePredictionExecution.run(); + Prediction completed = waitForPrediction(submitted.predictionId()); + assertThat(completed.status().terminationDetails().code()) + .isEqualTo(PredictionStatus.TerminationCode.SUCCESS); + assertThat(completed.snapshotId()).isEqualTo(pinnedSnapshot); + assertThat(completed.output()).isNotNull(); + assertThat(batchRead(catalog.getTable(completed.output().table().toIdentifier()))) + .containsExactly("+I[1, {\"text\":\"first\"}, null]"); + + resumePredictionExecution = predictionTestControl.pauseExecution(); + Prediction toCancel = + catalog.submitPrediction( + sourceIdentifier, + new PredictionSpec( + "cancel-reference", + modelIdentifier, + Collections.singletonMap("text", "content"), + Collections.singletonList("id"), + "Echo the input", + null, + Collections.emptyMap(), + "reference-cancel")); + waitForPredictionState(toCancel.predictionId(), PredictionStatus.State.RUNNING); + Prediction canceling = catalog.cancelPrediction(toCancel.predictionId()); + assertThat(canceling.status().state()).isEqualTo(PredictionStatus.State.TERMINATING); + resumePredictionExecution.run(); + Prediction canceled = waitForPrediction(toCancel.predictionId()); + assertThat(canceled.status().terminationDetails().code()) + .isEqualTo(PredictionStatus.TerminationCode.CANCELED); + assertThat(canceled.output()).isNull(); + } + + @Test + void testConcurrentIdempotentPredictionSubmissionsReturnSameJob() throws Exception { + PredictionFixture fixture = createPredictionFixture("prediction_concurrent_same"); + PredictionSpec spec = fixture.spec("same-request", "same-token"); + Runnable resumePredictionExecution = predictionTestControl.pauseExecution(); + predictionTestControl.synchronizeNextSubmissions(2); + ExecutorService submissions = Executors.newFixedThreadPool(2); + try { + Future first = + submissions.submit(() -> catalog.submitPrediction(fixture.source, spec)); + Future second = + submissions.submit(() -> catalog.submitPrediction(fixture.source, spec)); + + assertThat(first.get(10, TimeUnit.SECONDS).predictionId()) + .isEqualTo(second.get(10, TimeUnit.SECONDS).predictionId()); + } finally { + resumePredictionExecution.run(); + submissions.shutdownNow(); + } + } + + @Test + void testConcurrentConflictingPredictionSubmissionsReturnConflict() throws Exception { + PredictionFixture fixture = createPredictionFixture("prediction_concurrent_conflict"); + PredictionSpec firstSpec = fixture.spec("first-request", "conflict-token"); + PredictionSpec secondSpec = fixture.spec("second-request", "conflict-token"); + Runnable resumePredictionExecution = predictionTestControl.pauseExecution(); + predictionTestControl.synchronizeNextSubmissions(2); + ExecutorService submissions = Executors.newFixedThreadPool(2); + try { + List> results = + Arrays.asList( + submissions.submit( + () -> catalog.submitPrediction(fixture.source, firstSpec)), + submissions.submit( + () -> catalog.submitPrediction(fixture.source, secondSpec))); + int accepted = 0; + int conflicts = 0; + for (Future result : results) { + try { + result.get(10, TimeUnit.SECONDS); + accepted++; + } catch (ExecutionException e) { + assertThat(e.getCause()).isInstanceOf(AlreadyExistsException.class); + conflicts++; + } + } + assertThat(accepted).isEqualTo(1); + assertThat(conflicts).isEqualTo(1); + } finally { + resumePredictionExecution.run(); + submissions.shutdownNow(); + } + } + + @Test + void testCancelAfterOutputTableCreationDoesNotPublishTable() throws Exception { + PredictionFixture fixture = createPredictionFixture("prediction_cancel_after_create"); + Runnable resume = predictionTestControl.pauseAfterOutputCreated(); + Prediction submitted = + catalog.submitPrediction( + fixture.source, fixture.spec("cancel-after-create", "cancel-create-token")); + + predictionTestControl.awaitOutputCreated(); + assertThat(catalog.cancelPrediction(submitted.predictionId()).status().state()) + .isEqualTo(PredictionStatus.State.TERMINATING); + resume.run(); + + assertCanceledWithoutOutputTable(fixture, submitted.predictionId()); + } + + @Test + void testCancelAfterOutputCommitDoesNotPublishTable() throws Exception { + PredictionFixture fixture = createPredictionFixture("prediction_cancel_after_commit"); + Runnable resume = predictionTestControl.pauseAfterOutputCommitted(); + Prediction submitted = + catalog.submitPrediction( + fixture.source, fixture.spec("cancel-after-commit", "cancel-commit-token")); + + predictionTestControl.awaitOutputCommitted(); + assertThat(catalog.cancelPrediction(submitted.predictionId()).status().state()) + .isEqualTo(PredictionStatus.State.TERMINATING); + resume.run(); + + assertCanceledWithoutOutputTable(fixture, submitted.predictionId()); + } + + @Test + void testPredictionDerivesMergeKeyFromPrimaryKey() throws Exception { + PredictionFixture fixture = createPredictionFixture("prediction_derived_merge_key"); + Prediction submitted = + catalog.submitPrediction( + fixture.source, + fixture.specWithoutMergeKey("derived-merge-key", "derived-key-token")); + + Prediction completed = waitForPrediction(submitted.predictionId()); + assertThat(completed.status().terminationDetails().code()) + .isEqualTo(PredictionStatus.TerminationCode.SUCCESS); + assertThat(batchRead(catalog.getTable(completed.output().table().toIdentifier()))) + .containsExactlyInAnyOrder( + "+I[1, {\"text\":\"first\"}, null]", "+I[2, {\"text\":\"second\"}, null]"); + } + + @Test + void testPredictionRejectsDerivedMergeKeyUsingReservedOutputName() throws Exception { + PredictionFixture fixture = + createPredictionFixture("prediction_reserved_derived_key", "response"); + + assertThatThrownBy( + () -> + catalog.submitPrediction( + fixture.source, + fixture.specWithoutMergeKey( + "reserved-derived-key", "reserved-key-token"))) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining("reserved output name"); + } + + @Test + void testPredictionRejectsUnsupportedScanOptions() throws Exception { + PredictionFixture fixture = createPredictionFixture("prediction_scan_options", "id", 2); + for (Map.Entry option : + Arrays.asList( + new AbstractMap.SimpleEntry<>(CoreOptions.SCAN_BUCKET.key(), "0"), + new AbstractMap.SimpleEntry<>( + CoreOptions.STREAM_SCAN_MODE.key(), "file-monitor"))) { + PredictionSpec spec = + new PredictionSpec( + "restricted-" + option.getKey(), + fixture.model, + Collections.singletonMap("text", "content"), + Collections.singletonList("id"), + "Echo the input", + null, + Collections.singletonMap(option.getKey(), option.getValue()), + "restricted-" + option.getKey() + "-token"); + + assertThatThrownBy(() -> catalog.submitPrediction(fixture.source, spec)) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining(option.getKey()); + } + + PredictionFixture configured = + createPredictionFixture( + "prediction_stream_scan_mode", + "id", + 1, + Collections.singletonMap( + CoreOptions.STREAM_SCAN_MODE.key(), "file-monitor")); + assertThatThrownBy( + () -> + catalog.submitPrediction( + configured.source, + configured.spec( + "configured-stream-scan-mode", + "configured-stream-scan-mode-token"))) + .isInstanceOf(BadRequestException.class) + .hasMessageContaining(CoreOptions.STREAM_SCAN_MODE.key()); + } + + @Test + void testPredictionPublishesEmptyResultTable() throws Exception { + PredictionFixture fixture = createPredictionFixture("prediction_empty_result"); + FileStoreTable source = (FileStoreTable) catalog.getTable(fixture.source); + commitRows( + source, + GenericRow.ofKind(RowKind.DELETE, 1, BinaryString.fromString("first")), + GenericRow.ofKind(RowKind.DELETE, 2, BinaryString.fromString("second"))); + assertThat(batchRead(source)).isEmpty(); + + Prediction submitted = + catalog.submitPrediction( + fixture.source, fixture.spec("empty-result", "empty-result-token")); + Prediction completed = waitForPrediction(submitted.predictionId()); + + assertThat(completed.status().terminationDetails().code()) + .isEqualTo(PredictionStatus.TerminationCode.SUCCESS); + FileStoreTable output = + (FileStoreTable) catalog.getTable(completed.output().table().toIdentifier()); + assertThat(output.snapshotManager().latestSnapshot()).isNotNull(); + assertThat(output.snapshotManager().latestSnapshot().totalRecordCount()).isZero(); + assertThat(batchRead(output)).isEmpty(); + } + + private PredictionFixture createPredictionFixture(String database) throws Exception { + return createPredictionFixture(database, "id"); + } + + private PredictionFixture createPredictionFixture(String database, String primaryKey) + throws Exception { + return createPredictionFixture(database, primaryKey, 1); + } + + private PredictionFixture createPredictionFixture( + String database, String primaryKey, int buckets) throws Exception { + return createPredictionFixture(database, primaryKey, buckets, Collections.emptyMap()); + } + + private PredictionFixture createPredictionFixture( + String database, String primaryKey, int buckets, Map tableOptions) + throws Exception { + Identifier source = Identifier.create(database, "clips"); + Identifier model = Identifier.create(database, "echo_model"); + catalog.createDatabase(database, false); + Schema.Builder schema = + Schema.newBuilder() + .column(primaryKey, DataTypes.INT()) + .column("content", DataTypes.STRING()) + .primaryKey(primaryKey) + .option(CoreOptions.BUCKET.key(), Integer.toString(buckets)); + tableOptions.forEach(schema::option); + catalog.createTable(source, schema.build(), false); + commitRows( + (FileStoreTable) catalog.getTable(source), + GenericRow.of(1, BinaryString.fromString("first")), + GenericRow.of(2, BinaryString.fromString("second"))); + ProviderModelDefinition definition = + new ProviderModelDefinition( + "reference", + "echo", + Collections.singletonList(new ModelField("text", "STRING", null)), + Collections.singletonList(new ModelField("response", "STRING", null)), + Collections.emptyMap()); + catalog.createModel( + model, new Model(model, "Deterministic reference model", definition), false); + return new PredictionFixture(source, model); + } + + private void assertCanceledWithoutOutputTable(PredictionFixture fixture, String predictionId) + throws Exception { + Prediction canceled = waitForPrediction(predictionId); + assertThat(canceled.status().terminationDetails().code()) + .isEqualTo(PredictionStatus.TerminationCode.CANCELED); + assertThat(canceled.output()).isNull(); + String outputName = fixture.source.getObjectName() + "__" + predictionId.replace('-', '_'); + assertThat(catalog.listTables(fixture.source.getDatabaseName())).doesNotContain(outputName); + } + + private static final class PredictionFixture { + + private final Identifier source; + private final Identifier model; + + private PredictionFixture(Identifier source, Identifier model) { + this.source = source; + this.model = model; + } + + private PredictionSpec spec(String name, String idempotencyToken) { + return spec(name, Collections.singletonList("id"), idempotencyToken); + } + + private PredictionSpec specWithoutMergeKey(String name, String idempotencyToken) { + return spec(name, null, idempotencyToken); + } + + private PredictionSpec spec( + String name, List mergeKeyColumns, String idempotencyToken) { + return new PredictionSpec( + name, + model, + Collections.singletonMap("text", "content"), + mergeKeyColumns, + "Echo the input", + null, + Collections.emptyMap(), + idempotencyToken); + } + } + + private Prediction waitForPrediction(String predictionId) throws Exception { + long deadline = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(10); + while (System.currentTimeMillis() < deadline) { + Prediction prediction = catalog.getPrediction(predictionId); + if (prediction.status().isTerminal()) { + return prediction; + } + Thread.sleep(10L); + } + throw new AssertionError("Prediction did not terminate: " + predictionId); + } + + private void waitForPredictionState(String predictionId, PredictionStatus.State expectedState) + throws Exception { + long deadline = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(10); + while (System.currentTimeMillis() < deadline) { + if (catalog.getPrediction(predictionId).status().state() == expectedState) { + return; + } + Thread.sleep(10L); + } + throw new AssertionError("Prediction did not reach " + expectedState + ": " + predictionId); + } + @Test public void testAlterTableRethrowsForeignTypedNotFound() throws Exception { // a 404 typed other than TABLE/COLUMN must not be swallowed into reported success @@ -2002,6 +2386,7 @@ private RESTCatalog initCatalogUtil( this.config = new ConfigResponse(defaultConf, ImmutableMap.of()); restCatalogServer = new RESTCatalogServer(dataPath, this.authProvider, this.config, restWarehouse); + predictionTestControl = restCatalogServer.predictionTestControl(); restCatalogServer.start(); for (Map.Entry entry : this.authMap.entrySet()) { options.set(entry.getKey(), entry.getValue()); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogModelHandler.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogModelHandler.java index 2ab0b27a7d82..afdd42f8455b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogModelHandler.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogModelHandler.java @@ -31,6 +31,8 @@ import okhttp3.mockwebserver.MockResponse; +import javax.annotation.Nullable; + import java.util.Comparator; import java.util.List; import java.util.Map; @@ -103,6 +105,11 @@ void removeDatabase(String databaseName) { .equals(databaseName)); } + @Nullable + Model getModel(Identifier identifier) { + return modelStore.get(identifier.getFullName()); + } + private MockResponse list(String databaseName, Map parameters) { int maxResults = maxResults(parameters); String pageToken = parameters.get(PAGE_TOKEN); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java new file mode 100644 index 000000000000..d1e8a860501d --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionHandler.java @@ -0,0 +1,804 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.Snapshot; +import org.apache.paimon.catalog.Database; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.Prediction; +import org.apache.paimon.catalog.PredictionOutput; +import org.apache.paimon.catalog.PredictionSource; +import org.apache.paimon.catalog.PredictionSpec; +import org.apache.paimon.catalog.PredictionStatus; +import org.apache.paimon.catalog.TableMetadata; +import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; +import org.apache.paimon.reader.RecordReaderIterator; +import org.apache.paimon.rest.requests.SubmitPredictionRequest; +import org.apache.paimon.rest.responses.ErrorResponse; +import org.apache.paimon.rest.responses.GetPredictionResponse; +import org.apache.paimon.rest.responses.ListPredictionsResponse; +import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.sink.BatchTableCommit; +import org.apache.paimon.table.sink.BatchTableWrite; +import org.apache.paimon.table.sink.BatchWriteBuilder; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.utils.InternalRowUtils; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.core.JsonProcessingException; + +import okhttp3.mockwebserver.MockResponse; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; + +import static org.apache.paimon.rest.RESTApi.MAX_RESULTS; +import static org.apache.paimon.rest.RESTApi.PAGE_TOKEN; +import static org.apache.paimon.rest.RESTCatalogServerUtils.parseRequest; + +/** Executable prediction endpoints for the local REST catalog reference integration. */ +final class RESTCatalogPredictionHandler implements AutoCloseable { + + private static final String DATABASE = "database"; + private static final String TABLE = "table"; + private static final String NAME = "name"; + private static final String STATE = "state"; + + private final RESTCatalogServer server; + private final RESTFileSystemCatalog catalog; + private final RESTCatalogModelHandler modelHandler; + private final Map databaseStore; + private final Map tableMetadataStore; + private final String predictionsPath; + private final Map jobs = new ConcurrentHashMap<>(); + private final Map jobsByIdempotencyToken = new ConcurrentHashMap<>(); + private final AtomicLong nextId = new AtomicLong(); + private final ExecutorService executor; + + private volatile CountDownLatch executionGate = new CountDownLatch(0); + private volatile ExecutionGate outputCreatedGate = ExecutionGate.open(); + private volatile ExecutionGate outputCommittedGate = ExecutionGate.open(); + private volatile @Nullable CyclicBarrier submissionBarrier; + + RESTCatalogPredictionHandler( + RESTCatalogServer server, + RESTFileSystemCatalog catalog, + RESTCatalogModelHandler modelHandler, + Map databaseStore, + Map tableMetadataStore, + String predictionsPath) { + this.server = server; + this.catalog = catalog; + this.modelHandler = modelHandler; + this.databaseStore = databaseStore; + this.tableMetadataStore = tableMetadataStore; + this.predictionsPath = predictionsPath; + this.executor = + Executors.newSingleThreadExecutor( + runnable -> { + Thread thread = + new Thread(runnable, "paimon-reference-prediction-executor"); + thread.setDaemon(true); + return thread; + }); + } + + MockResponse handle( + String method, String resourcePath, String data, Map parameters) + throws Exception { + if (predictionsPath.equals(resourcePath)) { + if ("POST".equals(method)) { + return submit(data); + } else if ("GET".equals(method)) { + return list(parameters); + } + return new MockResponse().setResponseCode(404); + } + + String suffix = resourcePath.substring((predictionsPath + "/").length()); + String[] segments = suffix.split("/"); + String predictionId = RESTUtil.decodeString(segments[0]); + if (segments.length == 1 && "GET".equals(method)) { + return get(predictionId); + } + if (segments.length == 2 && "cancel".equals(segments[1]) && "POST".equals(method)) { + return cancel(predictionId); + } + return new MockResponse().setResponseCode(404); + } + + void pauseExecution() { + executionGate = new CountDownLatch(1); + } + + void resumeExecution() { + executionGate.countDown(); + } + + void synchronizeNextSubmissions(int parties) { + submissionBarrier = new CyclicBarrier(parties); + } + + void pauseAfterOutputCreated() { + outputCreatedGate = ExecutionGate.paused(); + } + + void awaitOutputCreated() throws InterruptedException { + outputCreatedGate.awaitReached(); + } + + void resumeAfterOutputCreated() { + outputCreatedGate.resume(); + } + + void pauseAfterOutputCommitted() { + outputCommittedGate = ExecutionGate.paused(); + } + + void awaitOutputCommitted() throws InterruptedException { + outputCommittedGate.awaitReached(); + } + + void resumeAfterOutputCommitted() { + outputCommittedGate.resume(); + } + + private MockResponse submit(String data) throws Exception { + SubmitPredictionRequest request = parseRequest(data, SubmitPredictionRequest.class); + PredictionJob existing = jobsByIdempotencyToken.get(request.getIdempotencyToken()); + if (existing != null) { + return idempotencyResponse(existing, data); + } + + Identifier source = request.getSource().toIdentifier(); + PredictionSpec spec = request.toPredictionSpec(); + MockResponse validationError = validate(source, spec); + if (validationError != null) { + return validationError; + } + + FileStoreTable sourceTable = (FileStoreTable) catalog.getTable(source); + long snapshotId = resolveSnapshot(sourceTable, spec); + String predictionId = String.format("prediction-%020d", nextId.incrementAndGet()); + PredictionJob job = + new PredictionJob( + predictionId, request.getSource(), spec, snapshotId, data, executionGate); + awaitSubmissionBarrier(); + PredictionJob selected = + jobsByIdempotencyToken.compute( + request.getIdempotencyToken(), + (token, current) -> { + if (current != null) { + return current; + } + jobs.put(predictionId, job); + try { + executor.submit(() -> execute(job)); + return job; + } catch (RuntimeException e) { + jobs.remove(predictionId, job); + throw e; + } + }); + return selected == job + ? response(new GetPredictionResponse(job.snapshot()), 202) + : idempotencyResponse(selected, data); + } + + private MockResponse idempotencyResponse(PredictionJob existing, String data) { + if (existing.requestPayload.equals(data)) { + return response(new GetPredictionResponse(existing.snapshot()), 202); + } + return error( + ErrorResponse.RESOURCE_TYPE_PREDICTION, + existing.predictionId, + "idempotencyToken is already used by another prediction", + 409); + } + + private void awaitSubmissionBarrier() throws Exception { + CyclicBarrier barrier = submissionBarrier; + if (barrier != null) { + barrier.await(10, TimeUnit.SECONDS); + submissionBarrier = null; + } + } + + @Nullable + private MockResponse validate(Identifier source, PredictionSpec spec) { + if (!databaseStore.containsKey(source.getDatabaseName())) { + return error( + ErrorResponse.RESOURCE_TYPE_DATABASE, + source.getDatabaseName(), + "Database does not exist", + 404); + } + if (!tableMetadataStore.containsKey(source.getFullName())) { + return error( + ErrorResponse.RESOURCE_TYPE_TABLE, + source.getObjectName(), + "Table does not exist", + 404); + } + Identifier model = spec.model(); + if (!databaseStore.containsKey(model.getDatabaseName())) { + return error( + ErrorResponse.RESOURCE_TYPE_DATABASE, + model.getDatabaseName(), + "Database does not exist", + 404); + } + if (modelHandler.getModel(model) == null) { + return error( + ErrorResponse.RESOURCE_TYPE_MODEL, + model.getObjectName(), + "Model does not exist", + 404); + } + if (spec.where() != null) { + return error(null, null, "The reference provider does not evaluate where", 501); + } + + TableSchema schema; + try { + schema = ((FileStoreTable) catalog.getTable(source)).schema(); + } catch (Exception e) { + return error( + ErrorResponse.RESOURCE_TYPE_TABLE, + source.getObjectName(), + "Table does not exist", + 404); + } + List fields = schema.fieldNames(); + List unsupportedScanOptions = + unsupportedScanOptions(schema.options(), spec.options()); + if (!unsupportedScanOptions.isEmpty()) { + return error( + null, + null, + "Prediction does not support scan options: " + unsupportedScanOptions, + 400); + } + for (String input : spec.inputs().values()) { + if (!fields.contains(input)) { + return error( + ErrorResponse.RESOURCE_TYPE_COLUMN, + input, + "Prediction input column does not exist", + 404); + } + } + for (String key : mergeKeyColumns(schema, spec)) { + if (!fields.contains(key)) { + return error( + ErrorResponse.RESOURCE_TYPE_COLUMN, + key, + "Prediction merge key column does not exist", + 404); + } + if ("response".equals(key) || "error".equals(key)) { + return error(null, null, "mergeKeyColumns use a reserved output name", 400); + } + } + return null; + } + + private long resolveSnapshot(FileStoreTable table, PredictionSpec spec) { + CoreOptions options = new CoreOptions(spec.options()); + Long snapshotId = options.scanSnapshotId(); + if (snapshotId != null) { + Snapshot snapshot = table.snapshotManager().snapshot(snapshotId); + if (snapshot == null) { + throw new IllegalArgumentException("Snapshot " + snapshotId + " does not exist"); + } + return snapshot.id(); + } + String tagName = options.scanTagName(); + if (tagName != null) { + return table.tagManager().getOrThrow(tagName).id(); + } + Snapshot latest = table.snapshotManager().latestSnapshot(); + if (latest == null) { + throw new IllegalArgumentException("The prediction source has no snapshot"); + } + return latest.id(); + } + + private void execute(PredictionJob job) { + @Nullable Identifier preparedOutput = null; + try { + if (!job.markRunning()) { + job.completeCanceled(); + return; + } + job.executionGate.await(); + checkCanceled(job); + preparedOutput = executePrediction(job); + Identifier output = preparedOutput; + if (!job.publishSuccess(output, () -> publishOutput(output))) { + cleanupOutput(output); + job.completeCanceled(); + } + } catch (InterruptedException e) { + cleanupOutput(job.stagingOutput()); + job.completeCanceled(); + Thread.currentThread().interrupt(); + } catch (Throwable t) { + cleanupOutput(preparedOutput == null ? job.stagingOutput() : preparedOutput); + if (job.isTerminating()) { + job.completeCanceled(); + } else { + job.completeFailure( + t.getMessage() == null ? t.getClass().getSimpleName() : t.getMessage()); + } + } + } + + private Identifier executePrediction(PredictionJob job) throws Exception { + Identifier sourceIdentifier = job.source.toIdentifier(); + FileStoreTable source = (FileStoreTable) catalog.getTable(sourceIdentifier); + Map scanOptions = new HashMap<>(job.spec.options()); + scanOptions.remove(CoreOptions.SCAN_TAG_NAME.key()); + scanOptions.put(CoreOptions.SCAN_SNAPSHOT_ID.key(), Long.toString(job.snapshotId)); + source = source.copy(scanOptions); + + Identifier outputIdentifier = + Identifier.create( + sourceIdentifier.getDatabaseName(), + sourceIdentifier.getObjectName() + + "__" + + job.predictionId.replace('-', '_')); + List mergeKeyColumns = mergeKeyColumns(source.schema(), job.spec); + Schema outputSchema = outputSchema(source.schema(), mergeKeyColumns); + catalog.createTable(outputIdentifier, outputSchema, false); + job.setStagingOutput(outputIdentifier); + FileStoreTable output = (FileStoreTable) catalog.getTable(outputIdentifier); + outputCreatedGate.reachAndAwait(); + checkCanceled(job); + + Map fieldIndexes = fieldIndexes(source.schema()); + BatchWriteBuilder writeBuilder = output.newBatchWriteBuilder(); + ReadBuilder readBuilder = source.newReadBuilder(); + try (RecordReaderIterator rows = + new RecordReaderIterator<>( + readBuilder + .newRead() + .createReader(readBuilder.newScan().plan().splits())); + BatchTableWrite write = writeBuilder.newWrite(); + BatchTableCommit commit = writeBuilder.newCommit()) { + while (rows.hasNext()) { + checkCanceled(job); + InternalRow sourceRow = rows.next(); + write.write( + toOutputRow( + sourceRow, + source.schema(), + fieldIndexes, + mergeKeyColumns, + job.spec)); + } + checkCanceled(job); + commit.commit(write.prepareCommit()); + } + + outputCommittedGate.reachAndAwait(); + checkCanceled(job); + return outputIdentifier; + } + + private void publishOutput(Identifier identifier) { + try { + FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); + Snapshot snapshot = table.snapshotManager().latestSnapshot(); + registerOutputTable(identifier, table); + server.setTableSnapshot( + identifier, + snapshot, + snapshot.totalRecordCount(), + 0L, + 0L, + snapshot.timeMillis()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void cleanupOutput(@Nullable Identifier identifier) { + if (identifier == null) { + return; + } + tableMetadataStore.remove(identifier.getFullName()); + try { + catalog.dropTable(identifier, true); + } catch (Exception ignored) { + // Best-effort cleanup for a test reference provider. + } + } + + private void registerOutputTable(Identifier identifier, FileStoreTable table) { + TableSchema schema = table.schema(); + Map options = new HashMap<>(schema.options()); + options.put(CoreOptions.PATH.key(), catalog.getTableLocation(identifier).toString()); + tableMetadataStore.put( + identifier.getFullName(), + new TableMetadata(schema.copy(options), false, UUID.randomUUID().toString())); + } + + private static Schema outputSchema(TableSchema source, List mergeKeyColumns) { + Schema.Builder builder = + Schema.newBuilder().option(CoreOptions.SNAPSHOT_IGNORE_EMPTY_COMMIT.key(), "false"); + Map fields = + source.fields().stream().collect(Collectors.toMap(DataField::name, field -> field)); + for (String name : mergeKeyColumns) { + builder.column(name, fields.get(name).type()); + } + return builder.column("response", DataTypes.STRING()) + .column("error", DataTypes.STRING()) + .build(); + } + + private static List unsupportedScanOptions( + Map tableOptions, Map requestOptions) { + List unsupported = new ArrayList<>(); + collectUnsupportedScanOptions(tableOptions, unsupported); + collectUnsupportedScanOptions(requestOptions, unsupported); + return unsupported; + } + + private static void collectUnsupportedScanOptions( + Map options, List unsupported) { + for (String key : options.keySet()) { + if ((key.startsWith("scan.") + || key.startsWith("incremental-") + || CoreOptions.STREAM_SCAN_MODE.key().equals(key)) + && !CoreOptions.SCAN_SNAPSHOT_ID.key().equals(key) + && !CoreOptions.SCAN_TAG_NAME.key().equals(key) + && !unsupported.contains(key)) { + unsupported.add(key); + } + } + } + + private static List mergeKeyColumns(TableSchema source, PredictionSpec spec) { + return spec.mergeKeyColumns().isEmpty() ? source.primaryKeys() : spec.mergeKeyColumns(); + } + + private static Map fieldIndexes(TableSchema schema) { + Map indexes = new HashMap<>(); + List fields = schema.fieldNames(); + for (int i = 0; i < fields.size(); i++) { + indexes.put(fields.get(i), i); + } + return indexes; + } + + private static InternalRow toOutputRow( + InternalRow sourceRow, + TableSchema sourceSchema, + Map fieldIndexes, + List mergeKeyColumns, + PredictionSpec spec) + throws JsonProcessingException { + Object[] values = new Object[mergeKeyColumns.size() + 2]; + int outputIndex = 0; + for (String key : mergeKeyColumns) { + int sourceIndex = fieldIndexes.get(key); + DataType type = sourceSchema.fields().get(sourceIndex).type(); + values[outputIndex++] = + InternalRowUtils.copy(InternalRowUtils.get(sourceRow, sourceIndex, type), type); + } + + Map response = new LinkedHashMap<>(); + for (Map.Entry input : spec.inputs().entrySet()) { + int sourceIndex = fieldIndexes.get(input.getValue()); + DataType type = sourceSchema.fields().get(sourceIndex).type(); + Object value = InternalRowUtils.get(sourceRow, sourceIndex, type); + response.put(input.getKey(), value == null ? null : value.toString()); + } + values[outputIndex] = BinaryString.fromString(RESTApi.toJson(response)); + values[outputIndex + 1] = null; + return GenericRow.of(values); + } + + private MockResponse get(String predictionId) { + PredictionJob job = jobs.get(predictionId); + return job == null + ? notFound(predictionId) + : response(new GetPredictionResponse(job.snapshot()), 200); + } + + private MockResponse cancel(String predictionId) { + PredictionJob job = jobs.get(predictionId); + if (job == null) { + return notFound(predictionId); + } + job.cancel(); + return response(new GetPredictionResponse(job.snapshot()), 200); + } + + private MockResponse list(Map parameters) { + List matching = + jobs.values().stream() + .filter(job -> matches(job, parameters)) + .sorted(Comparator.comparing(PredictionJob::id).reversed()) + .collect(Collectors.toList()); + String pageToken = parameters.get(PAGE_TOKEN); + if (pageToken != null) { + matching = + matching.stream() + .filter(job -> job.id().compareTo(pageToken) < 0) + .collect(Collectors.toList()); + } + int maxResults = maxResults(parameters); + boolean hasMore = matching.size() > maxResults; + List page = hasMore ? matching.subList(0, maxResults) : matching; + List responses = + page.stream() + .map(job -> new GetPredictionResponse(job.snapshot())) + .collect(Collectors.toList()); + String nextPageToken = hasMore ? page.get(page.size() - 1).id() : null; + return response(new ListPredictionsResponse(responses, nextPageToken), 200); + } + + private static boolean matches(PredictionJob job, Map parameters) { + Prediction prediction = job.snapshot(); + return matches(parameters.get(DATABASE), prediction.source().database()) + && matches(parameters.get(TABLE), prediction.source().table()) + && matches(parameters.get(NAME), prediction.name()) + && matches(parameters.get(STATE), prediction.status().state().name()); + } + + private static boolean matches(@Nullable String expected, @Nullable String actual) { + return expected == null || expected.equals(actual); + } + + private static int maxResults(Map parameters) { + String value = parameters.get(MAX_RESULTS); + if (value == null) { + return RESTCatalogServer.DEFAULT_MAX_RESULTS; + } + int requested = Integer.parseInt(value); + return requested <= 0 + ? RESTCatalogServer.DEFAULT_MAX_RESULTS + : Math.min(requested, RESTCatalogServer.DEFAULT_MAX_RESULTS); + } + + private static void checkCanceled(PredictionJob job) throws InterruptedException { + if (job.isTerminating() || Thread.currentThread().isInterrupted()) { + throw new InterruptedException("Prediction was canceled"); + } + } + + private static MockResponse notFound(String predictionId) { + return error( + ErrorResponse.RESOURCE_TYPE_PREDICTION, + predictionId, + "Prediction does not exist", + 404); + } + + private static MockResponse error( + @Nullable String resourceType, + @Nullable String resourceName, + String message, + int status) { + return response(new ErrorResponse(resourceType, resourceName, message, status), status); + } + + private static MockResponse response(Object response, int status) { + try { + return new MockResponse() + .setResponseCode(status) + .setBody(RESTApi.toJson(response)) + .addHeader("Content-Type", "application/json"); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() { + executionGate.countDown(); + outputCreatedGate.resume(); + outputCommittedGate.resume(); + executor.shutdownNow(); + try { + executor.awaitTermination(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + private static final class ExecutionGate { + + private final CountDownLatch reached; + private final CountDownLatch resume; + + private ExecutionGate(int count) { + this.reached = new CountDownLatch(count); + this.resume = new CountDownLatch(count); + } + + private static ExecutionGate open() { + return new ExecutionGate(0); + } + + private static ExecutionGate paused() { + return new ExecutionGate(1); + } + + private void reachAndAwait() throws InterruptedException { + reached.countDown(); + resume.await(); + } + + private void awaitReached() throws InterruptedException { + if (!reached.await(10, TimeUnit.SECONDS)) { + throw new AssertionError("Prediction execution did not reach the expected phase"); + } + } + + private void resume() { + resume.countDown(); + } + } + + private static final class PredictionJob { + + private final String predictionId; + private final PredictionSource source; + private final PredictionSpec spec; + private final long snapshotId; + private final String requestPayload; + private final CountDownLatch executionGate; + + private PredictionStatus status = + new PredictionStatus(PredictionStatus.State.PENDING, null); + private @Nullable PredictionOutput output; + private @Nullable Long startTime; + private @Nullable Long endTime; + private @Nullable Identifier stagingOutput; + + private PredictionJob( + String predictionId, + PredictionSource source, + PredictionSpec spec, + long snapshotId, + String requestPayload, + CountDownLatch executionGate) { + this.predictionId = predictionId; + this.source = source; + this.spec = spec; + this.snapshotId = snapshotId; + this.requestPayload = requestPayload; + this.executionGate = executionGate; + } + + private synchronized String id() { + return predictionId; + } + + private synchronized boolean markRunning() { + if (status.state() != PredictionStatus.State.PENDING) { + return false; + } + status = new PredictionStatus(PredictionStatus.State.RUNNING, null); + startTime = System.currentTimeMillis(); + return true; + } + + private synchronized boolean publishSuccess( + Identifier outputIdentifier, Runnable publisher) { + if (status.state() != PredictionStatus.State.RUNNING) { + return false; + } + publisher.run(); + output = new PredictionOutput(new PredictionSource(outputIdentifier)); + status = + new PredictionStatus( + PredictionStatus.State.TERMINATED, + new PredictionStatus.TerminationDetails( + PredictionStatus.TerminationCode.SUCCESS, null)); + endTime = System.currentTimeMillis(); + return true; + } + + private synchronized void completeFailure(String message) { + if (status.isTerminal()) { + return; + } + status = + new PredictionStatus( + PredictionStatus.State.TERMINATED, + new PredictionStatus.TerminationDetails( + PredictionStatus.TerminationCode.FAILED, message)); + endTime = System.currentTimeMillis(); + } + + private synchronized void completeCanceled() { + if (status.isTerminal()) { + return; + } + status = + new PredictionStatus( + PredictionStatus.State.TERMINATED, + new PredictionStatus.TerminationDetails( + PredictionStatus.TerminationCode.CANCELED, null)); + endTime = System.currentTimeMillis(); + } + + private synchronized void setStagingOutput(Identifier stagingOutput) { + this.stagingOutput = stagingOutput; + } + + private synchronized @Nullable Identifier stagingOutput() { + return stagingOutput; + } + + private synchronized boolean isTerminating() { + return status.state() == PredictionStatus.State.TERMINATING; + } + + private synchronized void cancel() { + if (status.isTerminal() || status.state() == PredictionStatus.State.TERMINATING) { + return; + } + if (status.state() == PredictionStatus.State.PENDING) { + completeCanceled(); + } else { + status = new PredictionStatus(PredictionStatus.State.TERMINATING, null); + } + } + + private synchronized Prediction snapshot() { + return new Prediction( + predictionId, + source, + spec.name(), + status, + snapshotId, + spec.model(), + output, + startTime, + endTime); + } + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java new file mode 100644 index 000000000000..67b0f165cbc4 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTest.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.PagedList; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.Prediction; +import org.apache.paimon.catalog.PredictionSpec; +import org.apache.paimon.catalog.PredictionStatus; +import org.apache.paimon.options.Options; +import org.apache.paimon.rest.exceptions.NoSuchResourceException; + +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** REST catalog tests for durable asynchronous predictions. */ +class RESTCatalogPredictionTest { + + @Test + void testPredictionLifecycleUsesSharedRestContract() throws Exception { + try (MockWebServer server = new MockWebServer()) { + server.start(); + Catalog catalog = catalog(server); + Identifier table = Identifier.create("quality", "clips"); + + server.enqueue(json(predictionJson("PENDING"), 202)); + Prediction submitted = catalog.submitPrediction(table, spec()); + assertThat(submitted.predictionId()).isEqualTo("prediction-1"); + RecordedRequest submit = server.takeRequest(10, TimeUnit.SECONDS); + assertThat(submit.getPath()).isEqualTo("/v1/catalog/predictions"); + Map request = RESTApi.fromJson(submit.getBody().readUtf8(), Map.class); + assertThat(request.get("idempotencyToken")).isInstanceOf(String.class); + assertThat((String) request.get("idempotencyToken")).isNotBlank(); + assertThat(request) + .doesNotContainKeys("resultColumns", "mergeKeyColumns", "outputSchema"); + assertThat((Map) request.get("source")) + .containsEntry("database", "quality") + .containsEntry("table", "clips"); + + server.enqueue( + json( + "{\"predictions\":[" + + predictionJson("RUNNING") + + "],\"nextPageToken\":\"next\"}", + 200)); + PagedList page = + catalog.listPredictions( + table, "daily-quality", PredictionStatus.State.RUNNING, 10, "page-1"); + assertThat(page.getElements()) + .extracting(Prediction::predictionId) + .containsExactly("prediction-1"); + assertThat(page.getNextPageToken()).isEqualTo("next"); + String listPath = server.takeRequest(10, TimeUnit.SECONDS).getPath(); + assertThat(listPath) + .contains("database=quality") + .contains("table=clips") + .contains("name=daily-quality") + .contains("state=RUNNING") + .contains("maxResults=10") + .contains("pageToken=page-1"); + + server.enqueue(json(terminatedPredictionJson("CANCELED"), 200)); + PredictionStatus cancelled = catalog.cancelPrediction("prediction/1").status(); + assertThat(cancelled.state()).isEqualTo(PredictionStatus.State.TERMINATED); + assertThat(cancelled.terminationDetails().code()) + .isEqualTo(PredictionStatus.TerminationCode.CANCELED); + assertThat(server.takeRequest(10, TimeUnit.SECONDS).getPath()) + .isEqualTo("/v1/catalog/predictions/prediction%2F1/cancel"); + } + } + + @Test + void testPredictionNotFoundPreservesResourceType() throws Exception { + try (MockWebServer server = new MockWebServer()) { + server.start(); + Catalog catalog = catalog(server); + Identifier table = Identifier.create("quality", "clips"); + + server.enqueue(notFound("DATABASE", "models")); + assertThatThrownBy(() -> catalog.submitPrediction(table, spec())) + .isInstanceOf(Catalog.DatabaseNotExistException.class) + .hasMessage("Database models does not exist."); + + server.enqueue(notFound("MODEL", "qwen_vl")); + assertThatThrownBy(() -> catalog.submitPrediction(table, spec())) + .isInstanceOf(Catalog.ModelNotExistException.class); + + server.enqueue(notFound("TABLE", "clips")); + assertThatThrownBy(() -> catalog.submitPrediction(table, spec())) + .isInstanceOf(Catalog.TableNotExistException.class); + + server.enqueue(untypedNotFound()); + assertThatThrownBy(() -> catalog.submitPrediction(table, spec())) + .isInstanceOf(NoSuchResourceException.class); + + server.enqueue(notFound("PREDICTION", "prediction-1")); + assertThatThrownBy(() -> catalog.getPrediction("prediction-1")) + .isInstanceOf(Catalog.PredictionNotExistException.class); + + server.enqueue(untypedNotFound()); + assertThatThrownBy(() -> catalog.getPrediction("prediction-1")) + .isInstanceOf(NoSuchResourceException.class); + + server.enqueue(untypedNotFound()); + assertThatThrownBy(() -> catalog.cancelPrediction("prediction-1")) + .isInstanceOf(NoSuchResourceException.class); + } + } + + @Test + void testPredictionRejectsBranchAndSystemTable() throws Exception { + try (MockWebServer server = new MockWebServer()) { + server.start(); + Catalog catalog = catalog(server); + + assertThatThrownBy( + () -> + catalog.submitPrediction( + new Identifier("quality", "clips", "dev"), spec())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("branch"); + assertThatThrownBy( + () -> + catalog.submitPrediction( + new Identifier("quality", "clips", null, "snapshots"), + spec())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("system table"); + } + } + + private static PredictionSpec spec() { + return new PredictionSpec( + "daily-quality", + Identifier.create("quality", "qwen_vl"), + Collections.singletonMap("video", "content"), + null, + "Classify", + null, + Collections.emptyMap(), + null); + } + + private static Catalog catalog(MockWebServer server) { + Options options = new Options(); + options.set(RESTCatalogOptions.URI, server.url("/").toString()); + options.set(RESTCatalogOptions.TOKEN_PROVIDER, "bear"); + options.set(RESTCatalogOptions.TOKEN, "token"); + options.set(RESTCatalogInternalOptions.PREFIX, "catalog"); + return new RESTCatalog(CatalogContext.create(options), false); + } + + private static String predictionJson(String status) { + return "{\"predictionId\":\"prediction-1\",\"source\":{\"database\":\"quality\"," + + "\"table\":\"clips\"},\"name\":\"daily-quality\",\"status\":{\"state\":\"" + + status + + "\"},\"snapshotId\":7,\"model\":{\"database\":\"quality\"," + + "\"object\":\"qwen_vl\"}}"; + } + + private static String terminatedPredictionJson(String code) { + return "{\"predictionId\":\"prediction-1\",\"source\":{\"database\":\"quality\"," + + "\"table\":\"clips\"},\"name\":\"daily-quality\"," + + "\"status\":{\"state\":\"TERMINATED\",\"terminationDetails\":{\"code\":\"" + + code + + "\"}},\"snapshotId\":7,\"model\":{\"database\":\"quality\"," + + "\"object\":\"qwen_vl\"}," + + "\"output\":{\"table\":{" + + "\"database\":\"system\",\"table\":\"results\"}}," + + "\"startTime\":10,\"endTime\":20}"; + } + + private static MockResponse json(String body, int code) { + return new MockResponse() + .setResponseCode(code) + .setHeader("Content-Type", "application/json") + .setBody(body); + } + + private static MockResponse notFound(String resourceType, String resourceName) { + return json( + "{\"resourceType\":\"" + + resourceType + + "\",\"resourceName\":\"" + + resourceName + + "\",\"message\":\"not found\",\"code\":404}", + 404); + } + + private static MockResponse untypedNotFound() { + return json("{\"message\":\"route not found\",\"code\":404}", 404); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTestControl.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTestControl.java new file mode 100644 index 000000000000..3afc3ec25b29 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogPredictionTestControl.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +/** Controls prediction execution phases for deterministic REST integration tests. */ +final class RESTCatalogPredictionTestControl { + + private final RESTCatalogPredictionHandler handler; + + RESTCatalogPredictionTestControl(RESTCatalogPredictionHandler handler) { + this.handler = handler; + } + + Runnable pauseExecution() { + handler.pauseExecution(); + return handler::resumeExecution; + } + + void synchronizeNextSubmissions(int parties) { + handler.synchronizeNextSubmissions(parties); + } + + Runnable pauseAfterOutputCreated() { + handler.pauseAfterOutputCreated(); + return handler::resumeAfterOutputCreated; + } + + void awaitOutputCreated() throws InterruptedException { + handler.awaitOutputCreated(); + } + + Runnable pauseAfterOutputCommitted() { + handler.pauseAfterOutputCommitted(); + return handler::resumeAfterOutputCommitted; + } + + void awaitOutputCommitted() throws InterruptedException { + handler.awaitOutputCommitted(); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index 0009d1d0287c..847456a2a993 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -262,6 +262,7 @@ public class RESTCatalogServer { private final Map functionStore = new ConcurrentHashMap<>(); private final RESTCatalogModelHandler modelHandler = new RESTCatalogModelHandler(databaseStore, databaseLifecycleLocks); + private final RESTCatalogPredictionHandler predictionHandler; private final Map> columnAuthHandler = new HashMap<>(); private final Map> rowFilterAuthHandler = new HashMap<>(); private final Map> columnMaskingAuthHandler = new HashMap<>(); @@ -300,6 +301,14 @@ public RESTCatalogServer( throw new UncheckedIOException(e); } this.catalog = new RESTFileSystemCatalog(fileIO, warehousePath, catalogContext); + this.predictionHandler = + new RESTCatalogPredictionHandler( + this, + catalog, + modelHandler, + databaseStore, + tableMetadataStore, + resourcePaths.predictions()); Dispatcher dispatcher = initDispatcher(authProvider); MockWebServer mockWebServer = new MockWebServer(); mockWebServer.setDispatcher(dispatcher); @@ -322,9 +331,14 @@ public void setTableSchema(Identifier identifier, TableSchema schema) { } public void shutdown() throws IOException { + predictionHandler.close(); server.shutdown(); } + RESTCatalogPredictionTestControl predictionTestControl() { + return new RESTCatalogPredictionTestControl(predictionHandler); + } + public void setTableSnapshot( Identifier identifier, Snapshot snapshot, @@ -498,6 +512,10 @@ && isTableByIdRequest(request.getPath())) { } else if (StringUtils.startsWith( request.getPath(), resourcePaths.functions())) { return functionsHandle(parameters); + } else if (resourcePaths.predictions().equals(resourcePath) + || resourcePath.startsWith(resourcePaths.predictions() + "/")) { + return predictionHandler.handle( + restAuthParameter.method(), resourcePath, data, parameters); } else if (request.getPath().startsWith(databaseUri)) { String[] resources = resourcePath.substring((databaseUri + "/").length()).split("/"); diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/ResourcePathsTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/ResourcePathsTest.java index b6f0f39f38f5..e31a2899a1b5 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/ResourcePathsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/ResourcePathsTest.java @@ -71,4 +71,13 @@ public void testPoliciesAreNestedUnderAttachmentResource() { "/v1/catalog%2Fid/databases/sales+db/tables/orders%2Fall/policies/drop", paths.dropPolicy(table)); } + + @Test + public void testPredictionRejectsDotSegments() { + ResourcePaths paths = new ResourcePaths("paimon"); + assertThrows(IllegalArgumentException.class, () -> paths.prediction(".")); + assertThrows(IllegalArgumentException.class, () -> paths.prediction("..")); + assertThrows(IllegalArgumentException.class, () -> paths.cancelPrediction(".")); + assertThrows(IllegalArgumentException.class, () -> paths.cancelPrediction("..")); + } } diff --git a/paimon-python/pypaimon/api/api_response.py b/paimon-python/pypaimon/api/api_response.py index 67609852af66..5e45a9afb5c3 100644 --- a/paimon-python/pypaimon/api/api_response.py +++ b/paimon-python/pypaimon/api/api_response.py @@ -50,6 +50,7 @@ class ErrorResponse(RESTResponse): RESOURCE_TYPE_VIEW = "VIEW" RESOURCE_TYPE_FUNCTION = "FUNCTION" RESOURCE_TYPE_MODEL = "MODEL" + RESOURCE_TYPE_PREDICTION = "PREDICTION" RESOURCE_TYPE_COLUMN = "COLUMN" RESOURCE_TYPE_SNAPSHOT = "SNAPSHOT" RESOURCE_TYPE_TAG = "TAG" diff --git a/paimon-python/pypaimon/api/prediction.py b/paimon-python/pypaimon/api/prediction.py new file mode 100644 index 000000000000..036933677770 --- /dev/null +++ b/paimon-python/pypaimon/api/prediction.py @@ -0,0 +1,223 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import uuid +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from pypaimon.api.api_request import RESTRequest +from pypaimon.api.api_response import PagedResponse, RESTResponse +from pypaimon.catalog.prediction import ( + Prediction, + PredictionOutput as CatalogPredictionOutput, + PredictionSpec, + PredictionStatus, + _copy_string_list, + _copy_string_mapping, + _nonblank, +) +from pypaimon.common.json_util import json_field, optional_json_field +from pypaimon.common.identifier import Identifier + + +@dataclass +class PredictionSource: + database: str + table: str + + def __post_init__(self): + _nonblank(self.database, "database") + _nonblank(self.table, "table") + + @classmethod + def from_identifier(cls, identifier: Identifier) -> "PredictionSource": + return cls( + identifier.get_database_name(), identifier.get_object_name()) + + def to_identifier(self) -> Identifier: + return Identifier.create(self.database, self.table) + + +@dataclass +class PredictionOutput: + table: PredictionSource + + def __post_init__(self): + if not isinstance(self.table, PredictionSource): + raise ValueError("table must be a PredictionSource") + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "PredictionOutput": + table = data.get("table") + return cls( + table=(PredictionSource(**table) + if isinstance(table, dict) else table), + ) + + def to_catalog_output(self) -> CatalogPredictionOutput: + return CatalogPredictionOutput(table=self.table.to_identifier()) + + +@dataclass +class SubmitPredictionRequest(RESTRequest): + source: PredictionSource + model: Identifier + inputs: Dict[str, str] + instructions: str + merge_key_columns: Optional[List[str]] = optional_json_field( + "mergeKeyColumns", "non_null") + name: Optional[str] = optional_json_field("name", "non_null") + idempotency_token: Optional[str] = optional_json_field( + "idempotencyToken", "non_null") + where: Optional[str] = optional_json_field("where", "non_null") + options: Dict[str, str] = field(default_factory=dict) + + def __post_init__(self): + if not isinstance(self.model, Identifier): + raise ValueError("model must be an Identifier") + _nonblank(self.instructions, "instructions") + for value, field_name in ( + (self.name, "name"), + (self.where, "where"), + ): + if value is not None: + _nonblank(value, field_name) + if self.idempotency_token is None: + self.idempotency_token = uuid.uuid4().hex + else: + _nonblank(self.idempotency_token, "idempotency_token") + if len(self.idempotency_token) > 64: + raise ValueError( + "idempotency_token must not exceed 64 characters") + self.inputs = _copy_string_mapping(self.inputs, "inputs", True) + if self.merge_key_columns is not None: + self.merge_key_columns = _copy_string_list( + self.merge_key_columns, "merge_key_columns") + self.options = _copy_string_mapping(self.options, "options", False) + PredictionSpec( + model=self.model, + inputs=self.inputs, + instructions=self.instructions, + merge_key_columns=self.merge_key_columns or [], + name=self.name, + where=self.where, + options=self.options, + ) + + @classmethod + def from_spec( + cls, source: Identifier, spec: PredictionSpec + ) -> "SubmitPredictionRequest": + return cls( + source=PredictionSource.from_identifier(source), + name=spec.name, + model=spec.model, + inputs=spec.inputs, + instructions=spec.instructions, + merge_key_columns=spec.merge_key_columns or None, + idempotency_token=spec.idempotency_token, + where=spec.where, + options=spec.options, + ) + + +@dataclass +class CancelPredictionRequest(RESTRequest): + """Empty body for a prediction cancellation request.""" + + pass + + +@dataclass +class GetPredictionResponse(RESTResponse): + """Prediction metadata with UTC Unix epoch millisecond timestamps.""" + + prediction_id: str = json_field("predictionId") + source: PredictionSource = None + status: PredictionStatus = None + snapshot_id: int = json_field("snapshotId", default=0) + model: Identifier = None + name: Optional[str] = optional_json_field("name", "non_null") + output: Optional[PredictionOutput] = optional_json_field( + "output", "non_null") + start_time: Optional[int] = optional_json_field("startTime", "non_null") + end_time: Optional[int] = optional_json_field("endTime", "non_null") + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "GetPredictionResponse": + return cls( + prediction_id=data.get("predictionId"), + source=( + PredictionSource(**data.get("source")) + if isinstance(data.get("source"), dict) + else data.get("source") + ), + name=data.get("name"), + status=PredictionStatus.from_dict(data.get("status")), + snapshot_id=data.get("snapshotId"), + model=( + Identifier(**data.get("model")) + if isinstance(data.get("model"), dict) + else data.get("model") + ), + output=(PredictionOutput.from_dict(data.get("output")) + if isinstance(data.get("output"), dict) + else data.get("output")), + start_time=data.get("startTime"), + end_time=data.get("endTime"), + ) + + def to_prediction(self) -> Prediction: + return Prediction( + prediction_id=self.prediction_id, + source=self.source.to_identifier(), + name=self.name, + status=self.status, + snapshot_id=self.snapshot_id, + model=self.model, + output=(self.output.to_catalog_output() + if self.output is not None else None), + start_time=self.start_time, + end_time=self.end_time, + ) + + +@dataclass +class ListPredictionsResponse(PagedResponse[GetPredictionResponse]): + predictions: List[GetPredictionResponse] = field(default_factory=list) + next_page_token: Optional[str] = json_field("nextPageToken", default=None) + + def __post_init__(self): + if self.predictions is None: + self.predictions = [] + + def data(self) -> List[GetPredictionResponse]: + return self.predictions + + def get_next_page_token(self) -> Optional[str]: + return self.next_page_token + + @classmethod + def from_dict(cls, data: Dict[str, Any]) -> "ListPredictionsResponse": + return cls( + predictions=[ + GetPredictionResponse.from_dict(value) + if isinstance(value, dict) else value + for value in (data.get("predictions") or []) + ], + next_page_token=data.get("nextPageToken"), + ) diff --git a/paimon-python/pypaimon/api/resource_paths.py b/paimon-python/pypaimon/api/resource_paths.py index 807744132aee..7fa70f76c007 100644 --- a/paimon-python/pypaimon/api/resource_paths.py +++ b/paimon-python/pypaimon/api/resource_paths.py @@ -33,6 +33,7 @@ class ResourcePaths: FUNCTIONS = "functions" FUNCTION_DETAILS = "function-details" MODELS = "models" + PREDICTIONS = "predictions" TAGS = "tags" BRANCHES = "branches" FORWARD = "forward" @@ -132,6 +133,20 @@ def model(self, database_name: str, model_name: str) -> str: self.models(database_name), RESTUtil.encode_string(model_name) ) + def predictions(self) -> str: + return "{}/{}".format(self.base_path, self.PREDICTIONS) + + def prediction(self, prediction_id: str) -> str: + if not isinstance(prediction_id, str) or not prediction_id.strip(): + raise ValueError("prediction_id must be a non-empty string") + if prediction_id in (".", ".."): + raise ValueError("prediction_id must not be a dot segment") + return "{}/{}".format( + self.predictions(), RESTUtil.encode_string(prediction_id)) + + def cancel_prediction(self, prediction_id: str) -> str: + return "{}/cancel".format(self.prediction(prediction_id)) + def tags(self, database_name: str, table_name: str) -> str: return "{}/{}/{}/{}/{}/{}".format( self.base_path, self.DATABASES, RESTUtil.encode_string(database_name), diff --git a/paimon-python/pypaimon/api/rest_api.py b/paimon-python/pypaimon/api/rest_api.py index cdd608f29cde..84d4910aeb23 100755 --- a/paimon-python/pypaimon/api/rest_api.py +++ b/paimon-python/pypaimon/api/rest_api.py @@ -53,6 +53,10 @@ from pypaimon.api.auth import AuthProviderFactory, RESTAuthFunction from pypaimon.api.client import HttpClient from pypaimon.api.model import CreateModelRequest, GetModelResponse, ListModelsResponse +from pypaimon.api.prediction import ( + CancelPredictionRequest, GetPredictionResponse, ListPredictionsResponse, + SubmitPredictionRequest, +) from pypaimon.api.resource_paths import ResourcePaths from pypaimon.api.rest_exception import NoSuchResourceException from pypaimon.api.rest_util import RESTUtil @@ -61,6 +65,7 @@ from pypaimon.common.options import Options from pypaimon.common.options.config import CatalogOptions from pypaimon.common.identifier import Identifier +from pypaimon.catalog.prediction import PredictionState from pypaimon.management.data_policy import DataPolicy from pypaimon.management.list_permissions_request import \ ListPermissionsRequest @@ -785,6 +790,59 @@ def drop_model(self, identifier: Identifier) -> None: self.rest_auth_function, ) + def submit_prediction( + self, + request: SubmitPredictionRequest, + ) -> GetPredictionResponse: + return self.client.post_with_response_type( + self.resource_paths.predictions(), + request, + GetPredictionResponse, + self.rest_auth_function, + ) + + def get_prediction(self, prediction_id: str) -> GetPredictionResponse: + return self.client.get( + self.resource_paths.prediction(prediction_id), + GetPredictionResponse, + self.rest_auth_function, + ) + + def list_predictions( + self, + source: Optional[Identifier] = None, + name: Optional[str] = None, + state: Optional[PredictionState] = None, + max_results: Optional[int] = None, + page_token: Optional[str] = None, + ) -> PagedList[GetPredictionResponse]: + query_params = {} + if source is not None: + database_name, table_name = self._validate_resource_path(source) + self._put_query_parameter(query_params, "database", database_name) + self._put_query_parameter(query_params, "table", table_name) + self._put_query_parameter(query_params, "name", name) + self._put_query_parameter( + query_params, "state", state.value if state is not None else None) + if max_results is not None and max_results > 0: + query_params[self.MAX_RESULTS] = str(max_results) + self._put_query_parameter(query_params, self.PAGE_TOKEN, page_token) + response = self.client.get_with_params( + self.resource_paths.predictions(), + query_params, + ListPredictionsResponse, + self.rest_auth_function, + ) + return PagedList(response.predictions, response.next_page_token) + + def cancel_prediction(self, prediction_id: str) -> GetPredictionResponse: + return self.client.post_with_response_type( + self.resource_paths.cancel_prediction(prediction_id), + CancelPredictionRequest(), + GetPredictionResponse, + self.rest_auth_function, + ) + def auth_table_query(self, identifier: Identifier, select: Optional[List[str]]) -> AuthTableQueryResponse: database_name, table_name = self.__validate_identifier(identifier) request = AuthTableQueryRequest(select=select) diff --git a/paimon-python/pypaimon/catalog/catalog.py b/paimon-python/pypaimon/catalog/catalog.py index 9ea6a83074a0..0fab39ac6c4c 100644 --- a/paimon-python/pypaimon/catalog/catalog.py +++ b/paimon-python/pypaimon/catalog/catalog.py @@ -111,6 +111,43 @@ def drop_model( """ raise NotImplementedError("drop_model is not supported by this catalog.") + def submit_prediction( + self, + identifier: Union[str, Identifier], + spec: 'PredictionSpec', + ) -> 'Prediction': + """Submit an asynchronous prediction over a table snapshot.""" + raise NotImplementedError( + "submit_prediction is not supported by this catalog.") + + def get_prediction( + self, + prediction_id: str, + ) -> 'Prediction': + """Get one prediction by its unique id.""" + raise NotImplementedError( + "get_prediction is not supported by this catalog.") + + def list_predictions( + self, + source: Optional[Union[str, Identifier]] = None, + name: Optional[str] = None, + state: Optional['PredictionState'] = None, + max_results: Optional[int] = None, + page_token: Optional[str] = None, + ) -> PagedList['Prediction']: + """List predictions, optionally filtered by source table.""" + raise NotImplementedError( + "list_predictions is not supported by this catalog.") + + def cancel_prediction( + self, + prediction_id: str, + ) -> 'Prediction': + """Request cancellation and return the updated prediction.""" + raise NotImplementedError( + "cancel_prediction is not supported by this catalog.") + def alter_database(self, name: str, changes: List[PropertyChange]) -> None: """Alter database properties. diff --git a/paimon-python/pypaimon/catalog/catalog_exception.py b/paimon-python/pypaimon/catalog/catalog_exception.py index cb2105401157..934e720d6dae 100644 --- a/paimon-python/pypaimon/catalog/catalog_exception.py +++ b/paimon-python/pypaimon/catalog/catalog_exception.py @@ -135,6 +135,14 @@ def __init__(self, identifier: Identifier): super().__init__(f"Model {identifier.get_full_name()} already exists") +class PredictionNotExistException(CatalogException): + """Prediction not exist exception""" + + def __init__(self, prediction_id: str): + self.prediction_id = prediction_id + super().__init__(f"Prediction {prediction_id} does not exist") + + class ColumnNotExistException(CatalogException): """Column not exist exception""" diff --git a/paimon-python/pypaimon/catalog/prediction.py b/paimon-python/pypaimon/catalog/prediction.py new file mode 100644 index 000000000000..1d86206a44e7 --- /dev/null +++ b/paimon-python/pypaimon/catalog/prediction.py @@ -0,0 +1,236 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from dataclasses import dataclass, field +from enum import Enum +from typing import Dict, List, Optional, Union + +from pypaimon.common.identifier import Identifier +from pypaimon.common.json_util import optional_json_field +from pypaimon.common.options.core_options import CoreOptions + + +class PredictionState(str, Enum): + PENDING = "PENDING" + RUNNING = "RUNNING" + TERMINATING = "TERMINATING" + TERMINATED = "TERMINATED" + + +class PredictionTerminationCode(str, Enum): + SUCCESS = "SUCCESS" + SUCCESS_WITH_FAILURES = "SUCCESS_WITH_FAILURES" + FAILED = "FAILED" + CANCELED = "CANCELED" + + +@dataclass +class PredictionTerminationDetails: + """Result and diagnostic message available after termination.""" + + code: PredictionTerminationCode + message: Optional[str] = None + + def __post_init__(self): + if isinstance(self.code, str): + self.code = PredictionTerminationCode(self.code) + if not isinstance(self.code, PredictionTerminationCode): + raise ValueError("code must be a PredictionTerminationCode") + + +@dataclass +class PredictionStatus: + """Lifecycle and optional termination details of a prediction.""" + + state: PredictionState + termination_details: Optional[PredictionTerminationDetails] = \ + optional_json_field("terminationDetails", "non_null") + + def __post_init__(self): + if isinstance(self.state, str): + self.state = PredictionState(self.state) + if not isinstance(self.state, PredictionState): + raise ValueError("state must be a PredictionState") + if (self.state is PredictionState.TERMINATED + and self.termination_details is None): + raise ValueError( + "termination_details must be present for TERMINATED state") + accepts_details = self.state in { + PredictionState.TERMINATING, + PredictionState.TERMINATED, + } + if not accepts_details and self.termination_details is not None: + raise ValueError( + "termination_details must not be present before " + "termination starts") + if self.termination_details is not None and not isinstance( + self.termination_details, PredictionTerminationDetails): + raise ValueError( + "termination_details must be PredictionTerminationDetails") + + @classmethod + def from_dict(cls, value): + details = value.get("terminationDetails") + return cls( + state=value.get("state"), + termination_details=( + PredictionTerminationDetails(**details) + if details is not None else None), + ) + + def to_dict(self): + result = {"state": self.state.value} + if self.termination_details is not None: + result["terminationDetails"] = { + "code": self.termination_details.code.value, + } + if self.termination_details.message is not None: + result["terminationDetails"]["message"] = \ + self.termination_details.message + return result + + @property + def terminal(self) -> bool: + return self.state is PredictionState.TERMINATED + + +@dataclass +class PredictionSpec: + """Provider-neutral input for an asynchronous table prediction.""" + + model: Union[str, Identifier] + inputs: Dict[str, str] + instructions: str + merge_key_columns: List[str] = field(default_factory=list) + name: Optional[str] = None + where: Optional[str] = None + options: Dict[str, str] = field(default_factory=dict) + idempotency_token: Optional[str] = None + + def __post_init__(self): + if isinstance(self.model, str): + self.model = Identifier.from_string(self.model) + if not isinstance(self.model, Identifier): + raise ValueError("model must be an Identifier or qualified name") + _nonblank(self.model.get_database_name(), "model database") + _nonblank(self.model.get_object_name(), "model object") + _nonblank(self.instructions, "instructions") + for value, name in ( + (self.name, "name"), + (self.where, "where"), + (self.idempotency_token, "idempotency_token"), + ): + if value is not None: + _nonblank(value, name) + if (self.idempotency_token is not None + and len(self.idempotency_token) > 64): + raise ValueError( + "idempotency_token must not exceed 64 characters") + self.inputs = _copy_string_mapping(self.inputs, "inputs", True) + self.merge_key_columns = _copy_string_list( + self.merge_key_columns, "merge_key_columns") + self.options = _copy_string_mapping(self.options, "options", False) + snapshot_id = self.options.get(CoreOptions.SCAN_SNAPSHOT_ID.key()) + tag_name = self.options.get(CoreOptions.SCAN_TAG_NAME.key()) + if snapshot_id is not None: + try: + if int(snapshot_id) <= 0: + raise ValueError + except ValueError: + raise ValueError( + "scan.snapshot-id must be a positive integer") + if snapshot_id is not None and tag_name is not None: + raise ValueError( + "scan.snapshot-id and scan.tag-name are mutually exclusive") + + +@dataclass +class PredictionOutput: + """Managed result table of an asynchronous prediction.""" + + table: Identifier + + def __post_init__(self): + if not isinstance(self.table, Identifier): + raise ValueError("table must be an Identifier") + + +@dataclass +class Prediction: + """Durable metadata for one asynchronous prediction job. + + ``start_time`` and ``end_time`` are Unix epoch milliseconds in UTC. + """ + + prediction_id: str + source: Identifier + status: PredictionStatus + snapshot_id: int + model: Union[str, Identifier] + name: Optional[str] = None + output: Optional[PredictionOutput] = None + start_time: Optional[int] = None + end_time: Optional[int] = None + + def __post_init__(self): + _nonblank(self.prediction_id, "prediction_id") + if not isinstance(self.source, Identifier): + raise ValueError("source must be an Identifier") + if not isinstance(self.status, PredictionStatus): + raise ValueError("status must be a PredictionStatus") + if type(self.snapshot_id) is not int or self.snapshot_id <= 0: + raise ValueError("snapshot_id must be a positive integer") + if isinstance(self.model, str): + self.model = Identifier.from_string(self.model) + if not isinstance(self.model, Identifier): + raise ValueError("model must be an Identifier or qualified name") + if self.output is not None and not isinstance( + self.output, PredictionOutput): + raise ValueError("output must be a PredictionOutput") + for name in ("start_time", "end_time"): + value = getattr(self, name) + if value is not None and (type(value) is not int or value < 0): + raise ValueError( + "{} must be a non-negative integer".format(name)) + + +def _nonblank(value, name): + if not isinstance(value, str) or not value.strip(): + raise ValueError("{} must be a non-empty string".format(name)) + + +def _copy_string_mapping(values, name, required): + if not isinstance(values, dict) or (required and not values): + raise ValueError("{} must be a non-empty mapping".format(name)) + result = {} + for key, value in values.items(): + _nonblank(key, "{} key".format(name)) + _nonblank(value, "{} value".format(name)) + result[key] = value + return result + + +def _copy_string_list(values, name): + if not isinstance(values, list): + raise ValueError("{} must be a list".format(name)) + result = [] + for value in values: + _nonblank(value, "{} value".format(name)) + if value in result: + raise ValueError("{} must not contain duplicates".format(name)) + result.append(value) + return result diff --git a/paimon-python/pypaimon/catalog/rest/rest_catalog.py b/paimon-python/pypaimon/catalog/rest/rest_catalog.py index 6caf66564a05..2de8eca60d30 100644 --- a/paimon-python/pypaimon/catalog/rest/rest_catalog.py +++ b/paimon-python/pypaimon/catalog/rest/rest_catalog.py @@ -19,6 +19,7 @@ from typing import Any, Callable, Dict, List, Optional, Union from pypaimon.api.api_response import ErrorResponse, GetTableResponse, GetTagResponse, PagedList, Partition from pypaimon.api.model import CreateModelRequest +from pypaimon.api.prediction import SubmitPredictionRequest from pypaimon.api.rest_api import RESTApi from pypaimon.api.rest_permission_management import RESTPermissionManagement from pypaimon.api.rest_policy_management import RESTPolicyManagement @@ -38,12 +39,14 @@ TableIdNotExistException, TableIdNoPermissionException, FunctionNotExistException, FunctionAlreadyExistException, ModelNotExistException, ModelAlreadyExistException, + PredictionNotExistException, DefinitionAlreadyExistException, DefinitionNotExistException, TagNotExistException, TagAlreadyExistException, BranchNotExistException, BranchAlreadyExistException, ) from pypaimon.catalog.database import Database from pypaimon.catalog.model import Model, ModelDefinition +from pypaimon.catalog.prediction import Prediction, PredictionSpec, PredictionState from pypaimon.catalog.rest.property_change import PropertyChange from pypaimon.catalog.rest.rest_token_file_io import RESTTokenFileIO from pypaimon.catalog.rest.table_metadata import TableMetadata @@ -678,6 +681,81 @@ def drop_model( except NotImplementedException as e: raise NotImplementedError("REST Catalog does not support models") from e + def submit_prediction( + self, + identifier: Union[str, Identifier], + spec: PredictionSpec, + ) -> Prediction: + if not isinstance(identifier, Identifier): + identifier = Identifier.from_string(identifier) + check_not_branch(identifier, "submit_prediction") + check_not_system_table(identifier, "submit_prediction") + try: + return self.rest_api.submit_prediction( + SubmitPredictionRequest.from_spec(identifier, spec)).to_prediction() + except NoSuchResourceException as e: + if e.resource_type == ErrorResponse.RESOURCE_TYPE_DATABASE: + raise DatabaseNotExistException( + e.resource_name + or identifier.get_database_name()) from e + if e.resource_type == ErrorResponse.RESOURCE_TYPE_MODEL: + raise ModelNotExistException(spec.model) from e + if e.resource_type == ErrorResponse.RESOURCE_TYPE_TABLE: + raise TableNotExistException(identifier) from e + raise + except NotImplementedException as e: + raise NotImplementedError( + "REST Catalog does not support predictions") from e + + def get_prediction( + self, + prediction_id: str, + ) -> Prediction: + try: + return self.rest_api.get_prediction(prediction_id).to_prediction() + except NoSuchResourceException as e: + if e.resource_type == ErrorResponse.RESOURCE_TYPE_PREDICTION: + raise PredictionNotExistException(prediction_id) from e + raise + except NotImplementedException as e: + raise NotImplementedError( + "REST Catalog does not support predictions") from e + + def list_predictions( + self, + source: Optional[Union[str, Identifier]] = None, + name: Optional[str] = None, + state: Optional[PredictionState] = None, + max_results: Optional[int] = None, + page_token: Optional[str] = None, + ) -> PagedList[Prediction]: + if source is not None and not isinstance(source, Identifier): + source = Identifier.from_string(source) + try: + page = self.rest_api.list_predictions( + source, name, state, max_results, page_token) + return PagedList( + [value.to_prediction() for value in page.elements], + page.next_page_token, + ) + except NotImplementedException as e: + raise NotImplementedError( + "REST Catalog does not support predictions") from e + + def cancel_prediction( + self, + prediction_id: str, + ) -> Prediction: + try: + return self.rest_api.cancel_prediction(prediction_id).to_prediction() + except NoSuchResourceException as e: + if e.resource_type == ErrorResponse.RESOURCE_TYPE_PREDICTION: + raise PredictionNotExistException(prediction_id) from e + raise + except NotImplementedException as e: + raise NotImplementedError( + "REST Catalog does not support predictions") from e + # Tag CRUD: mirrors Java RESTCatalog tag handlers. def create_tag(self, identifier: Union[str, Identifier], tag_name: str, snapshot_id: Optional[int] = None, diff --git a/paimon-python/pypaimon/common/wait.py b/paimon-python/pypaimon/common/wait.py new file mode 100644 index 000000000000..c23cefdd4d19 --- /dev/null +++ b/paimon-python/pypaimon/common/wait.py @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import timedelta +from typing import Callable, Generic, TypeVar + + +T = TypeVar("T") + + +class Wait(Generic[T]): + """Handle for a long-running operation.""" + + def __init__(self, waiter: Callable, response=None, **kwargs): + self.response = response + self._waiter = waiter + self._bind = kwargs + + def __getattr__(self, name): + try: + return self._bind[name] + except KeyError: + raise AttributeError(name) from None + + def bind(self): + return dict(self._bind) + + def result( + self, + timeout: timedelta = timedelta(minutes=20), + callback: Callable[[T], None] = None, + ) -> T: + """Delegate waiting and return the final value. + + The timeout is checked between polling requests. An in-flight request + may finish after the deadline, and timing out does not cancel the + server-side operation. + """ + kwargs = self._bind.copy() + return self._waiter(callback=callback, timeout=timeout, **kwargs) diff --git a/paimon-python/pypaimon/multimodal/prediction_wait.py b/paimon-python/pypaimon/multimodal/prediction_wait.py new file mode 100644 index 000000000000..5f0d400edc92 --- /dev/null +++ b/paimon-python/pypaimon/multimodal/prediction_wait.py @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import random +import time +from datetime import timedelta +from typing import Callable, Optional + +from pypaimon.catalog.prediction import Prediction +from pypaimon.common.wait import Wait + + +class PredictionWait(Wait[Prediction]): + """Wait handle returned after submitting an asynchronous prediction.""" + + def __init__(self, catalog, prediction: Prediction): + super().__init__( + lambda prediction_id, timeout, callback: _wait_for_prediction( + catalog, prediction_id, timeout, callback), + response=prediction, + prediction_id=prediction.prediction_id, + ) + + +def _wait_for_prediction( + catalog, + prediction_id: str, + timeout: timedelta, + callback: Optional[Callable[[Prediction], None]], +) -> Prediction: + if not isinstance(timeout, timedelta): + raise ValueError("timeout must be a datetime.timedelta") + if timeout.total_seconds() <= 0: + raise ValueError("timeout must be positive") + + deadline = time.monotonic() + timeout.total_seconds() + attempt = 1 + while time.monotonic() < deadline: + prediction = catalog.get_prediction(prediction_id) + if time.monotonic() >= deadline: + break + if prediction.status.terminal: + return prediction + if callback is not None: + callback(prediction) + remaining = deadline - time.monotonic() + if remaining <= 0: + break + sleep_seconds = min(attempt, 10) + random.random() + time.sleep(min(sleep_seconds, remaining)) + attempt += 1 + + raise TimeoutError( + "Prediction {} did not finish within {}".format( + prediction_id, timeout)) diff --git a/paimon-python/pypaimon/multimodal/table.py b/paimon-python/pypaimon/multimodal/table.py index de39dcb5be20..cee8c5f474f1 100644 --- a/paimon-python/pypaimon/multimodal/table.py +++ b/paimon-python/pypaimon/multimodal/table.py @@ -111,6 +111,112 @@ def copy(self, options: dict) -> 'MultimodalTable': return MultimodalTable( self.catalog, raw_table.identifier.get_full_name(), raw_table) + def predict( + self, + *, + model, + inputs, + instructions, + merge_key_columns=None, + name=None, + where=None, + options=None, + idempotency_token=None): + """Submit an asynchronous prediction without reading rows locally. + + ``options`` uses Paimon dynamic table option names. Time travel uses + ``scan.snapshot-id`` or ``scan.tag-name``. + + The returned waiter exposes the prediction ID immediately. Calling + ``result()`` is optional and intended for interactive clients; durable + workflows should persist the ID and observe the prediction + asynchronously. + """ + from pypaimon.catalog.prediction import PredictionSpec + from pypaimon.multimodal.prediction_wait import PredictionWait + from pypaimon.table.file_store_table import FileStoreTable + + if not isinstance(self.raw_table, FileStoreTable): + raise NotImplementedError( + "Prediction requires a Paimon file store table") + table_options = self.raw_table.options.options.to_map() + allowed = {"scan.snapshot-id", "scan.tag-name"} + prediction_options = dict(options or {}) + + def unsupported_scan_option(key): + return key not in allowed and ( + key.startswith("scan.") + or key.startswith("incremental-") + or key == "stream-scan-mode") + + unsupported = [ + key for key in table_options + if unsupported_scan_option(key) + ] + unsupported.extend([ + key for key in prediction_options + if unsupported_scan_option(key) + ]) + if unsupported: + raise ValueError( + "Prediction does not support scan options: {}".format( + unsupported)) + for key in allowed: + table_value = table_options.get(key) + if table_value is None: + continue + request_value = prediction_options.get(key) + if request_value is not None and request_value != table_value: + raise ValueError( + "Conflicting prediction option: {}".format(key)) + prediction_options[key] = table_value + spec = PredictionSpec( + name=name, + model=model, + inputs=inputs, + instructions=instructions, + merge_key_columns=( + [] if merge_key_columns is None else merge_key_columns), + where=where, + options=prediction_options, + idempotency_token=idempotency_token, + ) + prediction = self.catalog.submit_prediction( + self.raw_table.identifier, spec) + return PredictionWait(self.catalog, prediction) + + def get_prediction(self, prediction_id): + """Return metadata for one asynchronous prediction.""" + return self.catalog.get_prediction(prediction_id) + + def cancel_prediction(self, prediction_id): + """Request cancellation and return a wait handle.""" + from pypaimon.multimodal.prediction_wait import PredictionWait + + prediction = self.catalog.cancel_prediction(prediction_id) + return PredictionWait(self.catalog, prediction) + + def list_predictions( + self, + *, + name=None, + state=None, + max_results=None, + page_token=None): + """List prediction metadata for this table.""" + from pypaimon.catalog.prediction import PredictionState + + if isinstance(state, str): + state = PredictionState(state) + page = self.catalog.list_predictions( + source=self.raw_table.identifier, + name=name, + state=state, + max_results=max_results, + page_token=page_token, + ) + return page + def add(self, data): arrow_table = _to_arrow_table(data, _target_schema(self.raw_table)) write_builder = self.raw_table.new_batch_write_builder() diff --git a/paimon-python/pypaimon/tests/rest/client_test.py b/paimon-python/pypaimon/tests/rest/client_test.py index f064cca8c3c7..4261c9ae4cf3 100644 --- a/paimon-python/pypaimon/tests/rest/client_test.py +++ b/paimon-python/pypaimon/tests/rest/client_test.py @@ -133,7 +133,7 @@ def test_session_request_receives_timeout_tuple(self): client._execute_request('GET', 'http://localhost:8080/x') self.assertTrue(req.called, "session.request must be invoked") - kwargs = req.call_args.kwargs + kwargs = req.call_args[1] self.assertEqual( kwargs.get('timeout'), client._timeout, "request must carry the timeout; Session.timeout is dead code") diff --git a/paimon-python/pypaimon/tests/rest/prediction_test.py b/paimon-python/pypaimon/tests/rest/prediction_test.py new file mode 100644 index 000000000000..c85c03fb3b63 --- /dev/null +++ b/paimon-python/pypaimon/tests/rest/prediction_test.py @@ -0,0 +1,536 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +import threading +import unittest +from datetime import timedelta +from http.server import BaseHTTPRequestHandler, HTTPServer +from unittest.mock import Mock, patch + +from pypaimon.api.api_response import ErrorResponse, PagedList +from pypaimon.api.rest_exception import NoSuchResourceException +from pypaimon.api.prediction import ( + GetPredictionResponse, + ListPredictionsResponse, + PredictionOutput as ApiPredictionOutput, + PredictionSource, + SubmitPredictionRequest, +) +from pypaimon.api.rest_api import RESTApi +from pypaimon.catalog.catalog_exception import DatabaseNotExistException +from pypaimon.catalog.prediction import ( + Prediction, + PredictionOutput, + PredictionState, + PredictionSpec, + PredictionStatus, + PredictionTerminationCode, + PredictionTerminationDetails, +) +from pypaimon.catalog.rest.rest_catalog import RESTCatalog +from pypaimon.common.identifier import Identifier +from pypaimon.common.json_util import JSON +from pypaimon.common.options.core_options import CoreOptions +from pypaimon.common.options.options import Options +from pypaimon.multimodal.prediction_wait import PredictionWait +from pypaimon.multimodal.table import MultimodalTable +from pypaimon.table.file_store_table import FileStoreTable + + +def _spec(**changes): + values = dict( + model="quality.qwen_vl", + inputs={"video": "content"}, + instructions="Classify", + ) + values.update(changes) + return PredictionSpec(**values) + + +def _prediction(status=None, **changes): + if status is None: + status = PredictionStatus(PredictionState.PENDING) + values = dict( + prediction_id="prediction-1", + source=Identifier.create("quality", "clips"), + name="daily-quality", + status=status, + snapshot_id=7, + model="quality.qwen_vl", + ) + values.update(changes) + return Prediction(**values) + + +def _response(status=None, **changes): + if status is None: + status = PredictionStatus(PredictionState.PENDING) + values = dict( + prediction_id="prediction-1", + source=PredictionSource("quality", "clips"), + name="daily-quality", + status=status, + snapshot_id=7, + model=Identifier.create("quality", "qwen_vl"), + ) + values.update(changes) + return GetPredictionResponse(**values) + + +class PredictionTest(unittest.TestCase): + + def test_request_and_response_wire_format(self): + request = SubmitPredictionRequest.from_spec( + Identifier.create("quality", "clips"), _spec()) + body = json.loads(JSON.to_json(request)) + self.assertTrue(body["idempotencyToken"]) + self.assertEqual( + body["idempotencyToken"], + json.loads(JSON.to_json(request))["idempotencyToken"], + ) + self.assertNotIn("resultColumns", body) + self.assertNotIn("mergeKeyColumns", body) + self.assertNotIn("prompt", body) + self.assertEqual("Classify", body["instructions"]) + self.assertNotIn("outputSchema", body) + self.assertEqual( + {"database": "quality", "table": "clips"}, body["source"]) + self.assertEqual( + {"database": "quality", "object": "qwen_vl"}, body["model"]) + self.assertNotIn("snapshotId", body) + self.assertNotIn("tagName", body) + + versioned = SubmitPredictionRequest.from_spec( + Identifier.create("quality", "clips"), + _spec(options={"scan.tag-name": "training"}), + ) + versioned_body = json.loads(JSON.to_json(versioned)) + self.assertEqual( + {"scan.tag-name": "training"}, versioned_body["options"]) + self.assertNotIn("tagName", versioned_body) + + keyed = SubmitPredictionRequest.from_spec( + Identifier.create("quality", "clips"), + _spec( + merge_key_columns=["clip_id"], + idempotency_token="retry-1", + ), + ) + keyed_body = json.loads(JSON.to_json(keyed)) + self.assertEqual(["clip_id"], keyed_body["mergeKeyColumns"]) + self.assertEqual("retry-1", keyed_body["idempotencyToken"]) + + response = JSON.from_json( + JSON.to_json(_response()), GetPredictionResponse) + self.assertEqual( + PredictionState.PENDING, response.to_prediction().status.state) + self.assertIsNone( + response.to_prediction().status.termination_details) + pending_body = json.loads(JSON.to_json(_response())) + for field in ( + "createdAt", "startedAt", "finishedAt", + "modelVersion", "outputTable", "output", "startTime", + "endTime"): + self.assertNotIn(field, pending_body) + terminated = PredictionStatus( + PredictionState.TERMINATED, + PredictionTerminationDetails( + PredictionTerminationCode.FAILED, + "Model request failed", + ), + ) + completed_response = _response( + terminated, + output=ApiPredictionOutput( + PredictionSource("system", "prediction_results")), + start_time=10, + end_time=20, + ) + terminated_body = json.loads(JSON.to_json(completed_response)) + self.assertEqual( + { + "state": "TERMINATED", + "terminationDetails": { + "code": "FAILED", + "message": "Model request failed", + }, + }, + terminated_body["status"], + ) + self.assertEqual(10, terminated_body["startTime"]) + self.assertEqual(20, terminated_body["endTime"]) + self.assertEqual( + {"table": {"database": "system", "table": "prediction_results"}}, + terminated_body["output"], + ) + for field in ( + "createdAt", "startedAt", "finishedAt", "modelVersion", + "outputTable"): + self.assertNotIn(field, terminated_body) + completed = JSON.from_json( + JSON.to_json(completed_response), GetPredictionResponse) + self.assertEqual( + terminated, + completed.status, + ) + self.assertEqual( + PredictionOutput( + Identifier.create("system", "prediction_results")), + completed.to_prediction().output, + ) + with self.assertRaises(ValueError): + ApiPredictionOutput.from_dict( + {"artifactUri": "oss://bucket/result"}) + page = JSON.from_json( + JSON.to_json(ListPredictionsResponse([_response()], "next")), + ListPredictionsResponse, + ) + self.assertEqual("prediction-1", page.predictions[0].prediction_id) + self.assertEqual("next", page.next_page_token) + + def test_validation(self): + invalid = [ + dict(model=""), + dict(inputs={}), + dict(inputs={"video": ""}), + dict(merge_key_columns=["id", "id"]), + dict(merge_key_columns="id"), + dict(instructions=" "), + dict(options={"scan.snapshot-id": "0"}), + dict(options={ + "scan.snapshot-id": "1", + "scan.tag-name": "v1", + }), + dict(options={"parallelism": 2}), + dict(idempotency_token="x" * 65), + ] + for changes in invalid: + with self.subTest(changes=changes), self.assertRaises(ValueError): + _spec(**changes) + with self.assertRaises(ValueError): + _spec(model=Identifier.create("", "qwen_vl")) + with self.assertRaises(ValueError): + PredictionStatus(PredictionState.TERMINATED) + with self.assertRaises(ValueError): + PredictionStatus( + PredictionState.RUNNING, + PredictionTerminationDetails( + PredictionTerminationCode.FAILED), + ) + + def test_terminating_does_not_require_final_result(self): + terminating = PredictionStatus(PredictionState.TERMINATING) + self.assertFalse(terminating.terminal) + for code in ( + PredictionTerminationCode.CANCELED, + PredictionTerminationCode.SUCCESS, + ): + with self.subTest(code=code): + terminated = PredictionStatus( + PredictionState.TERMINATED, + PredictionTerminationDetails(code), + ) + self.assertTrue(terminated.terminal) + self.assertEqual(code, terminated.termination_details.code) + + def test_structured_model_preserves_dots(self): + request = SubmitPredictionRequest.from_spec( + Identifier.create("quality", "clips"), + _spec(model=Identifier.create("quality.archive", "qwen.vl")), + ) + body = json.loads(JSON.to_json(request)) + + self.assertEqual( + {"database": "quality.archive", "object": "qwen.vl"}, + body["model"], + ) + + def test_table_submit_fixes_view_and_returns_waiter(self): + raw = Mock(spec=FileStoreTable) + raw.options = CoreOptions(Options({"scan.snapshot-id": "7"})) + raw.identifier = Identifier.create("quality", "clips") + catalog = Mock() + catalog.submit_prediction.return_value = _prediction() + table = MultimodalTable(catalog, "quality.clips", raw) + + waiter = table.predict( + name="daily-quality", + model="quality.qwen_vl", + inputs={"video": "content"}, + instructions="Classify", + idempotency_token="retry-1", + ) + + self.assertIsInstance(waiter, PredictionWait) + self.assertEqual("prediction-1", waiter.prediction_id) + sent = catalog.submit_prediction.call_args[0][1] + self.assertEqual({"scan.snapshot-id": "7"}, sent.options) + self.assertEqual([], sent.merge_key_columns) + self.assertEqual("retry-1", sent.idempotency_token) + raw.new_read_builder.assert_not_called() + + def test_table_submit_rejects_stream_scan_mode(self): + for table_options, request_options in ( + ({"stream-scan-mode": "file-monitor"}, None), + ({}, {"stream-scan-mode": "file-monitor"}), + ): + with self.subTest( + table_options=table_options, + request_options=request_options): + raw = Mock(spec=FileStoreTable) + raw.options = CoreOptions(Options(table_options)) + raw.identifier = Identifier.create("quality", "clips") + catalog = Mock() + table = MultimodalTable(catalog, "quality.clips", raw) + + with self.assertRaisesRegex(ValueError, "stream-scan-mode"): + table.predict( + model="quality.qwen_vl", + inputs={"video": "content"}, + instructions="Classify", + options=request_options, + ) + catalog.submit_prediction.assert_not_called() + + def test_table_queries_return_metadata_and_cancel_returns_waiter(self): + raw = Mock(spec=FileStoreTable) + raw.identifier = Identifier.create("quality", "clips") + catalog = Mock() + prediction = _prediction() + page = PagedList([prediction], "next") + catalog.get_prediction.return_value = prediction + catalog.list_predictions.return_value = page + catalog.cancel_prediction.return_value = prediction + table = MultimodalTable(catalog, "quality.clips", raw) + + self.assertIs(prediction, table.get_prediction("prediction-1")) + self.assertIs( + page, + table.list_predictions(state=PredictionState.PENDING), + ) + waiter = table.cancel_prediction("prediction-1") + self.assertIsInstance(waiter, PredictionWait) + self.assertEqual("prediction-1", waiter.prediction_id) + + @patch("pypaimon.multimodal.prediction_wait.random.random", return_value=0) + @patch("pypaimon.multimodal.prediction_wait.time.sleep") + def test_result_is_client_side_polling(self, sleep, random): + catalog = Mock() + catalog.get_prediction.side_effect = [ + _prediction(PredictionStatus(PredictionState.RUNNING)), + _prediction(PredictionStatus( + PredictionState.TERMINATED, + PredictionTerminationDetails( + PredictionTerminationCode.SUCCESS), + )), + ] + waiter = PredictionWait(catalog, _prediction()) + observed = [] + result = waiter.result( + timeout=timedelta(seconds=10), + callback=lambda prediction: observed.append( + prediction.status.state), + ) + self.assertEqual(PredictionState.TERMINATED, result.status.state) + self.assertEqual( + PredictionTerminationCode.SUCCESS, + result.status.termination_details.code, + ) + self.assertEqual([PredictionState.RUNNING], observed) + self.assertEqual(2, catalog.get_prediction.call_count) + sleep.assert_called_once_with(1) + + def test_result_returns_failed_and_canceled_predictions(self): + for code in ( + PredictionTerminationCode.FAILED, + PredictionTerminationCode.CANCELED, + ): + with self.subTest(code=code): + catalog = Mock() + catalog.get_prediction.return_value = _prediction( + PredictionStatus( + PredictionState.TERMINATED, + PredictionTerminationDetails(code, "terminal"), + )) + waiter = PredictionWait(catalog, _prediction()) + result = waiter.result(timeout=timedelta(seconds=1)) + self.assertEqual( + code, result.status.termination_details.code) + + @patch( + "pypaimon.multimodal.prediction_wait.time.monotonic", + side_effect=[0, 2], + ) + def test_result_timeout_does_not_cancel_prediction(self, monotonic): + catalog = Mock() + waiter = PredictionWait(catalog, _prediction()) + + with self.assertRaisesRegex( + TimeoutError, "Prediction prediction-1 did not finish"): + waiter.result(timeout=timedelta(seconds=1)) + + catalog.get_prediction.assert_not_called() + catalog.cancel_prediction.assert_not_called() + + @patch( + "pypaimon.multimodal.prediction_wait.time.monotonic", + side_effect=[0, 0, 0.2], + ) + def test_result_rejects_terminal_response_after_deadline(self, monotonic): + catalog = Mock() + catalog.get_prediction.return_value = _prediction( + PredictionStatus( + PredictionState.TERMINATED, + PredictionTerminationDetails( + PredictionTerminationCode.SUCCESS), + )) + waiter = PredictionWait(catalog, _prediction()) + + with self.assertRaisesRegex( + TimeoutError, "Prediction prediction-1 did not finish"): + waiter.result(timeout=timedelta(milliseconds=10)) + + catalog.get_prediction.assert_called_once_with("prediction-1") + + def test_wait_missing_attribute_uses_attribute_error(self): + waiter = PredictionWait(Mock(), _prediction()) + with self.assertRaises(AttributeError): + waiter.unknown_attribute + + def test_untyped_prediction_404_is_not_mapped_to_missing_resource(self): + catalog = object.__new__(RESTCatalog) + catalog.rest_api = Mock() + untyped = NoSuchResourceException(None, None, "route not found") + + catalog.rest_api.submit_prediction.side_effect = untyped + with self.assertRaises(NoSuchResourceException): + catalog.submit_prediction( + Identifier.create("quality", "clips"), _spec()) + + catalog.rest_api.get_prediction.side_effect = untyped + with self.assertRaises(NoSuchResourceException): + catalog.get_prediction("prediction-1") + + catalog.rest_api.cancel_prediction.side_effect = untyped + with self.assertRaises(NoSuchResourceException): + catalog.cancel_prediction("prediction-1") + + def test_submit_prediction_maps_database_404(self): + catalog = object.__new__(RESTCatalog) + catalog.rest_api = Mock() + catalog.rest_api.submit_prediction.side_effect = NoSuchResourceException( + ErrorResponse.RESOURCE_TYPE_DATABASE, + "models", + "database not found", + ) + + with self.assertRaises(DatabaseNotExistException) as context: + catalog.submit_prediction( + Identifier.create("quality", "clips"), _spec()) + + self.assertEqual("models", context.exception.database) + + def test_rest_round_trip_paths_and_filters(self): + calls = [] + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def _send(self, payload, status=200): + body = JSON.to_json(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_POST(self): + length = int(self.headers.get("Content-Length", "0")) + body = json.loads(self.rfile.read(length)) if length else None + calls.append(("POST", self.path, body)) + status = 202 if self.path.endswith("/predictions") else 200 + self._send(_response(), status) + + def do_GET(self): + calls.append(("GET", self.path, None)) + if "/predictions?" in self.path: + self._send(ListPredictionsResponse([_response()], "next")) + else: + self._send(_response()) + + server = HTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + api = RESTApi( + { + "uri": "http://127.0.0.1:%s" % server.server_port, + "token.provider": "bear", + "token": "test", + "prefix": "catalog", + }, + False, + ) + catalog = object.__new__(RESTCatalog) + catalog.rest_api = api + identifier = Identifier.create("a/b", "clip table") + + self.assertEqual("prediction-1", catalog.submit_prediction( + identifier, _spec()).prediction_id) + self.assertEqual( + "/v1/catalog/predictions", + calls[-1][1], + ) + self.assertEqual( + {"database": "a/b", "table": "clip table"}, + calls[-1][2]["source"], + ) + self.assertTrue(calls[-1][2]["idempotencyToken"]) + self.assertNotIn("mergeKeyColumns", calls[-1][2]) + + page = catalog.list_predictions( + identifier, + name="daily-quality", + state=PredictionState.RUNNING, + max_results=10, + page_token="p1", + ) + self.assertEqual("prediction-1", page.elements[0].prediction_id) + self.assertIn("name=daily-quality", calls[-1][1]) + self.assertIn("database=a%2Fb", calls[-1][1]) + self.assertIn("table=clip+table", calls[-1][1]) + self.assertIn("state=RUNNING", calls[-1][1]) + self.assertIn("maxResults=10", calls[-1][1]) + self.assertIn("pageToken=p1", calls[-1][1]) + + catalog.get_prediction("prediction/1") + self.assertTrue( + calls[-1][1].endswith("/predictions/prediction%2F1")) + catalog.cancel_prediction("prediction/1") + self.assertTrue( + calls[-1][1].endswith( + "/predictions/prediction%2F1/cancel")) + finally: + server.shutdown() + server.server_close() + thread.join() + + +if __name__ == "__main__": + unittest.main() diff --git a/paimon-python/pypaimon/tests/rest/resource_paths_test.py b/paimon-python/pypaimon/tests/rest/resource_paths_test.py index eccd72f68f86..731e90901b89 100644 --- a/paimon-python/pypaimon/tests/rest/resource_paths_test.py +++ b/paimon-python/pypaimon/tests/rest/resource_paths_test.py @@ -51,6 +51,15 @@ def test_database_and_table_names_are_url_encoded(self): "/v1/catalog%2Fid/databases/sales+db/table-details", paths.table_details("sales db")) self.assertEqual("/v1/catalog%2Fid/tables/id/a%2Fb+c", paths.table_by_id("a/b c")) + def test_prediction_rejects_dot_segments(self): + paths = ResourcePaths("paimon") + for prediction_id in (".", ".."): + with self.subTest(prediction_id=prediction_id): + with self.assertRaises(ValueError): + paths.prediction(prediction_id) + with self.assertRaises(ValueError): + paths.cancel_prediction(prediction_id) + def test_signature_matches_the_path_sent(self): received = [] diff --git a/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py b/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py index 72cbc13e3bf4..973f9e86fd31 100644 --- a/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py +++ b/paimon-python/pypaimon/tests/rest/rest_api_coverage_test.py @@ -27,6 +27,7 @@ from pypaimon.api.rest_api import RESTApi from pypaimon.api.model import CreateModelRequest +from pypaimon.api.prediction import PredictionSource, SubmitPredictionRequest from pypaimon.catalog.model import ModelField, ProviderModelDefinition from pypaimon.common.identifier import Identifier from pypaimon.function.function import FunctionImpl @@ -44,8 +45,13 @@ from pypaimon.snapshot.snapshot_commit import PartitionStatistics from pypaimon.table.instant import Instant -# RESTApi methods with no entry in dlf_openapi_actions; they are signed without an action. -WITHOUT_ACTION = frozenset() +# REST-only APIs with no DLF POP action; they are signed without x-acs-action. +WITHOUT_ACTION = frozenset({ + "cancel_prediction", + "get_prediction", + "list_predictions", + "submit_prediction", +}) _SPEC_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), *[os.pardir] * 4, "docs", "static") _SPEC_FILES = ("rest-catalog-open-api.yaml", "rest-management-open-api.yaml") @@ -63,6 +69,12 @@ [ModelField("video", "BLOB", "video")], ), ), + SubmitPredictionRequest: SubmitPredictionRequest( + source=PredictionSource("db", "t"), + model=Identifier.create("db", "qwen_vl"), + inputs={"video": "content"}, + instructions="classify", + ), str: "SELECT", bool: False, Identifier: Identifier.create("db", "t"),