diff --git a/CHANGELOG.md b/CHANGELOG.md index 021de916..1cf17652 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,21 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.22.0](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.22.0) - 2026-08-26 + +### Added +- **Run-free ("model v2") predictions.** Predictions can now be uploaded and read directly against a `Model`, with no `ModelRun` or `Dataset` involved — the concept is `(model, dataset_item) -> prediction`. New methods on `Model`: + - `Model.upload_predictions(predictions, update=False, batch_size=5000, ...)` — upserts predictions onto the model (`box` / `polygon` / `cuboid` only), targeting `model/{id}/predictions`, and reusing the existing `PredictionUploader` batching machinery. Synchronous only for now: `asynchronous=True` raises `NotImplementedError`. + - `Model.predictions_loc(dataset_item_id)`, `Model.predictions_refloc(reference_id)`, `Model.predictions_iloc(i)` — model-scoped reads returning the same shape as their `Dataset` equivalents. + - `Model.copy_predictions_from_run(model_run_id)` — synchronously backfills the run-free store from an existing model run, returning a dict `{model_id, model_run_ids, predictions_copied, predictions_skipped_unsupported}`. +- **Model-anchored benchmark evaluations.** `NucleusClient.create_benchmark_evaluation_v2()` accepts a `model_id` (a `prj_*` id or a `Model`) as an alternative to `model_run_id`; the model-anchored flow evaluates the model's run-free predictions and ignores model runs. Provide exactly one of the two. `EvaluationV2` now exposes an optional `model_id` field alongside `model_run_id`. + +### Changed +- The existing run-based prediction paths (`Dataset.upload_predictions`, `ModelRun.add_predictions`, `create_benchmark_evaluation_v2(model_run_id=...)`) are unchanged and continue to work; the model-centric methods are purely additive. + +### Fixed +- `Model.predictions_loc` / `predictions_refloc` / `predictions_iloc` now actually parse their responses. The run-free read endpoints return a flat `{"predictions": [...]}` list (each element carrying its own `"type"`), but `format_prediction_response` only understood the legacy type-keyed `{"annotations": {"box": [...]}}` shape, so these methods returned the raw payload unparsed instead of the documented `{"box": [...], "polygon": [...], "cuboid": [...]}` dict. + ## [0.21.2](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.21.2) - 2026-08-17 ### Added diff --git a/nucleus/__init__.py b/nucleus/__init__.py index e5a94c5a..b9bab5b9 100644 --- a/nucleus/__init__.py +++ b/nucleus/__init__.py @@ -149,6 +149,7 @@ MESSAGE_KEY, METADATA_KEY, METRIC_TYPE_KEY, + MODEL_ID_KEY, MODEL_IDS_KEY, MODEL_RUN_ID_KEY, MODEL_RUN_IDS_KEY, @@ -1562,8 +1563,9 @@ def finalize_benchmark(self, benchmark_id: str) -> Benchmark: def create_benchmark_evaluation_v2( self, benchmark_id: str, - model_run_id: str, + model_run_id: Optional[str] = None, *, + model_id: Optional[Union[str, Model]] = None, name: Optional[str] = None, rollup_groups: Optional[List[RollupGroup]] = None, allowed_label_matches: Optional[List[AllowedLabelMatch]] = None, @@ -1587,10 +1589,19 @@ def create_benchmark_evaluation_v2( being rejected. To give a run predictions across several datasets, use :meth:`Dataset.upload_predictions_for_model_run`. + The evaluation can be anchored on either a legacy model run + (``model_run_id``) or, for the run-free "model v2" flow, a model + (``model_id``). Provide exactly one; the model-anchored flow evaluates + the model's run-free predictions and ignores model runs entirely. + Parameters: benchmark_id: Benchmark id (``bm_*``). model_run_id: Model run id (``run_*``). It need not cover the benchmark's datasets — coverage may be partial, or empty. + Mutually exclusive with ``model_id``. + model_id: Model id (``prj_*``) or :class:`Model` to anchor the + evaluation on the model's run-free predictions instead of a + model run. Mutually exclusive with ``model_run_id``. name: Optional display name. rollup_groups: Optional rollup classes (the primary label configuration); each :class:`RollupGroup` maps raw labels @@ -1610,6 +1621,13 @@ def create_benchmark_evaluation_v2( Returns: :class:`EvaluationV2`: The created evaluation. """ + resolved_model_id = ( + model_id.id if isinstance(model_id, Model) else model_id + ) + if (resolved_model_id is None) == (model_run_id is None): + raise ValueError( + "Provide exactly one of model_run_id or model_id." + ) if preset is not None: if ( rollup_groups is None @@ -1635,7 +1653,11 @@ def create_benchmark_evaluation_v2( "Set at most one of rollup_groups, allowed_label_matches, " "or allowed_label_matches_id" ) - payload: Dict[str, Any] = {MODEL_RUN_ID_KEY: model_run_id} + payload: Dict[str, Any] = ( + {MODEL_ID_KEY: resolved_model_id} + if resolved_model_id is not None + else {MODEL_RUN_ID_KEY: model_run_id} + ) if name is not None: payload[NAME_KEY] = name if rollup_groups is not None: diff --git a/nucleus/evaluation_v2.py b/nucleus/evaluation_v2.py index 0d4847af..a70e8bd3 100644 --- a/nucleus/evaluation_v2.py +++ b/nucleus/evaluation_v2.py @@ -31,6 +31,7 @@ LABELS_KEY, LIMIT_KEY, MATCH_TYPE_KEY, + MODEL_ID_KEY, MODEL_PREDICTION_LABEL_CAMEL_KEY, MODEL_PREDICTION_LABEL_KEY, MODEL_RUN_ID_KEY, @@ -170,12 +171,13 @@ def _parse_rollup_groups(raw_groups: Any) -> Optional[List[RollupGroup]]: @dataclass class EvaluationV2: - """An Evaluation V2 run for a model run.""" + """An Evaluation V2 run for a model run or a run-free model.""" id: str - model_run_id: str + model_run_id: Optional[str] dataset_id: str status: str + model_id: Optional[str] = None name: Optional[str] = None temporal_workflow_id: Optional[str] = None error_message: Optional[str] = None @@ -202,9 +204,18 @@ def from_json( return cls( id=str(payload[ID_KEY]), - model_run_id=str(payload[MODEL_RUN_ID_KEY]), + model_run_id=( + str(payload[MODEL_RUN_ID_KEY]) + if payload.get(MODEL_RUN_ID_KEY) is not None + else None + ), dataset_id=str(payload[DATASET_ID_KEY]), status=str(payload[STATUS_KEY]), + model_id=( + str(payload[MODEL_ID_KEY]) + if payload.get(MODEL_ID_KEY) is not None + else None + ), name=payload.get(NAME_KEY), temporal_workflow_id=payload.get(TEMPORAL_WORKFLOW_ID_KEY), error_message=payload.get(ERROR_MESSAGE_KEY), diff --git a/nucleus/model.py b/nucleus/model.py index 18340571..d6bab803 100644 --- a/nucleus/model.py +++ b/nucleus/model.py @@ -1,10 +1,14 @@ -from typing import Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union import requests +from nucleus.annotation_uploader import PredictionUploader +from nucleus.utils import format_prediction_response + from .async_job import AsyncJob from .constants import ( METADATA_KEY, + MODEL_RUN_ID_KEY, MODEL_TAGS_KEY, MODEL_TRAINED_SLICE_IDS_KEY, NAME_KEY, @@ -17,6 +21,7 @@ BoxPrediction, CuboidPrediction, PolygonPrediction, + Prediction, SegmentationPrediction, ) @@ -229,6 +234,189 @@ def create_run( run.add_predictions(predictions) return run + def upload_predictions( + self, + predictions: List[Prediction], + update: bool = False, + asynchronous: bool = False, + batch_size: int = 5000, + remote_files_per_upload_request: int = 20, + local_files_per_upload_request: int = 10, + ) -> Dict[str, Any]: + """Uploads predictions directly to this model, with no model run. + + This is the run-free ("model v2") prediction path: predictions are tied + to the model itself as ``(model, dataset_item) -> prediction`` and are + upserted server-side. Each prediction identifies its target item by + ``dataset_item_id`` (the ``di_*`` id returned on exported items) or by + ``reference_id``, so a single model can hold predictions for items that + live in different datasets — no :class:`Dataset` or :class:`ModelRun` is + needed. Reads go through :meth:`predictions_loc`, + :meth:`predictions_refloc`, and :meth:`predictions_iloc`. + + Only ``box``, ``polygon``, and ``cuboid`` predictions are accepted on + this path. + + The legacy run-based path (:meth:`Dataset.upload_predictions` / + :meth:`ModelRun.add_predictions`) continues to work unchanged. + + Args: + predictions: List of prediction objects to upload. + update: If True, existing predictions for the same + (reference_id, annotation_id) are overwritten. If False, they + are skipped. Default is False. + asynchronous: Not yet supported for this path — passing True raises + :class:`NotImplementedError`. The upload always runs + synchronously. + batch_size: Number of predictions processed in each concurrent + batch. Default is 5000. If you get timeouts when uploading + geometric predictions, you can try lowering this batch size. + remote_files_per_upload_request: Number of remote files to upload in + each request. + local_files_per_upload_request: Number of local files to upload in + each request. The maximum is 10. + + Returns: + Payload describing the synchronous upload:: + + { + "model_id": str, + "predictions_processed": int, + "predictions_ignored": int, + } + """ + if asynchronous: + raise NotImplementedError( + "async is not yet supported for model prediction v2 uploads; " + "use asynchronous=False" + ) + + uploader = PredictionUploader( + client=self._client, + route=f"model/{self.id}/predictions", + ) + uploader.check_for_duplicate_ids(predictions) + + return uploader.upload( + annotations=predictions, + batch_size=batch_size, + update=update, + remote_files_per_upload_request=remote_files_per_upload_request, + local_files_per_upload_request=local_files_per_upload_request, + ) + + def predictions_loc(self, dataset_item_id: str): + """Fetches all of this model's predictions for a dataset item by its id. + + Model-scoped counterpart of :meth:`Dataset.prediction_loc` for the + run-free prediction path. + + Parameters: + dataset_item_id: Internally controlled id for the dataset item + (``di_*``). + + Returns: + Dictionary mapping prediction type to a list of prediction objects + for this model:: + + { + "box": List[BoxPrediction], + "polygon": List[PolygonPrediction], + "cuboid": List[CuboidPrediction], + } + """ + return format_prediction_response( + self._client.make_request( + payload=None, + route=f"model/{self.id}/predictions/loc/{dataset_item_id}", + requests_command=requests.get, + ) + ) + + def predictions_refloc(self, reference_id: str): + """Fetches all of this model's predictions for a dataset item by its reference id. + + Model-scoped counterpart of :meth:`Dataset.predictions_refloc` for the + run-free prediction path. + + Parameters: + reference_id: User-defined reference id of the dataset item. + + Returns: + Dictionary mapping prediction type to a list of prediction objects + for this model:: + + { + "box": List[BoxPrediction], + "polygon": List[PolygonPrediction], + "cuboid": List[CuboidPrediction], + } + """ + return format_prediction_response( + self._client.make_request( + payload=None, + route=f"model/{self.id}/predictions/refloc/{reference_id}", + requests_command=requests.get, + ) + ) + + def predictions_iloc(self, i: int): + """Fetches all of this model's predictions for a dataset item by its index. + + Model-scoped counterpart of :meth:`Dataset.predictions_iloc` for the + run-free prediction path. + + Parameters: + i: Absolute index of the dataset item. + + Returns: + Dictionary mapping prediction type to a list of prediction objects + for this model:: + + { + "box": List[BoxPrediction], + "polygon": List[PolygonPrediction], + "cuboid": List[CuboidPrediction], + } + """ + return format_prediction_response( + self._client.make_request( + payload=None, + route=f"model/{self.id}/predictions/iloc/{i}", + requests_command=requests.get, + ) + ) + + def copy_predictions_from_run(self, model_run_id: str) -> Dict[str, Any]: + """Copies predictions from a legacy v1 model run onto this model. + + Backfills the run-free ("model v2") prediction store for this model from + an existing :class:`ModelRun`, so predictions previously uploaded via the + run-based path become readable through :meth:`predictions_loc` and + friends. The source run is left untouched. + + Runs synchronously server-side and returns once the copy completes. + + Args: + model_run_id: Source model run id (``run_*``) to copy predictions + from. + + Returns: + Payload describing the copy:: + + { + "model_id": str, + "model_run_ids": List[str], + "predictions_copied": int, + "predictions_skipped_unsupported": int, + } + """ + return self._client.make_request( + {MODEL_RUN_ID_KEY: model_run_id}, + route=f"model/{self.id}/predictions/copyFromRun", + requests_command=requests.post, + ) + def evaluate(self, scenario_test_names: List[str]) -> AsyncJob: """Evaluates this on the specified Unit Tests. :: diff --git a/nucleus/model_run.py b/nucleus/model_run.py index aa626ca8..3f20d7ef 100644 --- a/nucleus/model_run.py +++ b/nucleus/model_run.py @@ -1,18 +1,29 @@ """ Model Runs are deprecated and will be removed in a future version of the python client. -It is now possible to upload model predictions without a need for creating a model run +It is now possible to upload model predictions without a need for creating a model run. -For example:: +The recommended run-free ("model v2") path uploads and reads predictions +directly on a :class:`~nucleus.model.Model` — the concept is +``(model, dataset_item) -> prediction``, with no model run or dataset:: import nucleus client = nucleus.NucleusClient(YOUR_SCALE_API_KEY) prediction_1 = nucleus.BoxPrediction(label="label", x=0, y=0, width=10, height=10, reference_id="1", confidence=0.9, class_pdf={'label': 0.9, 'other_label': 0.1}) prediction_2 = nucleus.BoxPrediction(label="label", x=0, y=0, width=10, height=10, reference_id="2", confidence=0.2, class_pdf={'label': 0.2, 'other_label': 0.8}) model = client.create_model(name="My Model", reference_id="My-CNN", metadata={"timestamp": "121012401"}) + + # Run-free upload / read, tied to the model itself: + model.upload_predictions([prediction_1, prediction_2]) + model.predictions_refloc("1") + +Benchmark evaluations can likewise anchor on the model directly, ignoring model +runs, via ``client.create_benchmark_evaluation_v2(benchmark_id, model_id=model.id)``. + +The older per-dataset path also remains available:: + response = dataset.upload_predictions(model, [prediction_1, prediction_2]) """ - from typing import List, Optional, Union import requests diff --git a/nucleus/utils.py b/nucleus/utils.py index d4db15dc..6fd870d7 100644 --- a/nucleus/utils.py +++ b/nucleus/utils.py @@ -47,6 +47,7 @@ SCALE_TASK_INFO_KEY, SCENE_KEY, SEGMENTATION_TYPE, + TYPE_KEY, ) from .dataset_item import DatasetItem from .prediction import ( @@ -131,10 +132,6 @@ def format_prediction_response( keyed by the type name. """ annotation_payload = response.get(ANNOTATIONS_KEY, None) - if not annotation_payload: - # An error occurred - return response - annotation_response = {} type_key_to_class: Dict[ str, Union[ @@ -156,6 +153,24 @@ def format_prediction_response( KEYPOINTS_TYPE: KeypointsPrediction, SEGMENTATION_TYPE: SegmentationPrediction, } + if not annotation_payload: + # Run-free ("model v2") reads (Model.predictions_loc / _refloc / + # _iloc) return a flat list under "predictions", each element carrying + # its own "type", rather than the type-keyed "annotations" dict. Group + # it into the same {type: [obj, ...]} shape those methods promise. + prediction_payload = response.get(PREDICTIONS_KEY, None) + if not prediction_payload: + # An error occurred, or there are no predictions. + return response + annotation_response: Dict[str, list] = {} + for prediction in prediction_payload: + type_key = prediction[TYPE_KEY] + type_class = type_key_to_class[type_key] + annotation_response.setdefault(type_key, []).append( + type_class.from_json(prediction) + ) + return annotation_response + annotation_response = {} for type_key in annotation_payload: type_class = type_key_to_class[type_key] annotation_response[type_key] = [ @@ -230,6 +245,7 @@ def format_scale_task_info_response(response: dict) -> Union[Dict, List[Dict]]: ret.append(row) return ret + # pylint: disable=too-many-branches,too-many-statements def convert_export_payload(api_payload, has_predictions: bool = False): """Helper function to convert raw JSON to API objects diff --git a/pyproject.toml b/pyproject.toml index 4901f914..5b47c0b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ ignore = ["E501", "E741", "E731", "F401"] # Easy ignore for getting it running [tool.poetry] name = "scale-nucleus" -version = "0.21.2" +version = "0.22.0" description = "The official Python client library for Nucleus, the Data Platform for AI" license = "MIT" authors = ["Scale AI Nucleus Team "]