From d602d5592a2f1b76613c5f7c835eafe43f116aaa Mon Sep 17 00:00:00 2001 From: "hotdata-automation[bot]" <267177015+hotdata-automation[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 03:09:01 +0000 Subject: [PATCH] feat(indexes): add algorithm and probe_fraction support --- CHANGELOG.md | 1 + docs/CreateIndexRequest.md | 5 +++- docs/IndexEntryResponse.md | 2 ++ docs/IndexInfoResponse.md | 2 ++ docs/JobResult.md | 2 ++ hotdata/models/create_index_request.py | 36 ++++++++++++++++++++++---- hotdata/models/index_entry_response.py | 10 ++++--- hotdata/models/index_info_response.py | 20 +++++++++++--- test/test_create_index_request.py | 3 +++ test/test_index_entry_response.py | 2 ++ test/test_index_info_response.py | 2 ++ test/test_job_result.py | 2 ++ test/test_list_indexes_response.py | 4 +++ 13 files changed, 79 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a19d8b4..844d433 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- feat(indexes): add algorithm and probe_fraction support - chore: add 403 forbidden response to endpoints - feat(jobs): add database_fork job type diff --git a/docs/CreateIndexRequest.md b/docs/CreateIndexRequest.md index 3093b79..de0eb9c 100644 --- a/docs/CreateIndexRequest.md +++ b/docs/CreateIndexRequest.md @@ -6,6 +6,7 @@ Request body for POST .../indexes One constraint spans the whole table rather t Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- +**algorithm** | **str** | How a vector index organises the vectors it searches. Omit this field for `hnsw`, which is the default. `hnsw` — builds a graph of the vectors and keeps it in memory. Searches are very fast, and the memory a search needs grows with the whole table, so a large enough table cannot be served at all. `ivf` — groups the vectors into clusters and reads only the clusters nearest the search. Searches are considerably slower than `hnsw`, and the memory a search needs follows how much of the index it reads rather than the size of the table, so a table far too large for `hnsw` can still be searched. It keeps a copy of the table's rows beside the vectors so a search is answered without reading the table; that copy is extra storage, and how much depends on `vector_precision`, which decides how compactly the copied vectors are held. Available for columns that already hold vectors, with the `l2` and `cosine` metrics. | [optional] **var_async** | **bool** | When true, create the index as a background job and return a job ID for polling. | [optional] [default to False] **async_after_ms** | **int** | If set (requires `async` = true), wait up to this many milliseconds for the index build to finish: if it completes in time the index is returned (201), otherwise a 202 with a job ID to poll. Must be between 1000 and the server maximum; a value out of that range, or set without `async` = true, is rejected with 400. | [optional] **columns** | **List[str]** | Columns to index. Required for all index types. | @@ -15,8 +16,10 @@ Name | Type | Description | Notes **index_name** | **str** | | **index_type** | **str** | Index type. `sorted` supports range queries, `bm25` full-text search, and `vector` similarity search. | [optional] [default to 'sorted'] **metric** | **str** | Distance metric for vector indexes: \"l2\", \"cosine\", or \"dot\". When omitted, defaults to \"l2\" for float array columns or the provider's preferred metric for text columns with auto-embedding. | [optional] +**nlist** | **int** | Number of clusters an `ivf` index divides the vectors into. More clusters means each one holds fewer vectors, so a search of the same effort reads less data. Omit this to let the number be chosen from the table's size. | [optional] **output_column** | **str** | Custom name for the generated embedding column. Defaults to `{column}_embedding`. | [optional] -**vector_precision** | **str** | How precisely a vector index stores each number of a vector. Lower precision shrinks the index so a larger table can be indexed within the same memory, and lets searches run on a smaller instance. Omit this field to store vectors at the same precision as the column, which is the default. The quality figures below come from one benchmark — 1536-dimension text embeddings, cosine distance, default search settings — and are a guide, not a guarantee. Other models, dimensions, distance metrics and data distributions behave differently, so measure on your own data before moving a production index to a lower precision. `float32` — on a `float64` column this halves the index. Widely used embedding models emit 32-bit values, so for those nothing is lost; vectors that genuinely carry more than 32 bits of precision will lose some. `float16` — half the memory of `float32`. In that benchmark its results matched `float32` to within 0.1 percentage points. `float8` — a quarter of the memory of `float32`. In that benchmark it scored about 4 percentage points below `float32`, and raising the search effort did not close the gap, so treat the reduction as permanent for a given index. `float64` — accepted only for a column that already holds double-precision values; it cannot add precision the stored data does not have. Changing this means dropping the index and creating it again. It affects only the index: the table's own values are never altered, and text columns indexed with a generated embedding are not re-embedded. | [optional] +**probe_fraction** | **float** | How much of an `ivf` index a search reads, as a fraction greater than 0 and at most 1. Higher finds more of the true nearest neighbours and takes longer. This is a fraction rather than a number of clusters on purpose: the same number of clusters is a different share of the index whenever `nlist` changes, and results would quietly get worse. Omit this for the server's default. | [optional] +**vector_precision** | **str** | How precisely a vector index stores each number of a vector. Lower precision shrinks the index so a larger table can be indexed within the same memory, and lets searches run on a smaller instance. For an `ivf` index it also shrinks what every search reads, because a search reads part of that stored copy. Omit this field to get each algorithm's own default: an `hnsw` index stores vectors at the same precision as the column, and an `ivf` index stores them as `int8`. The quality figures below come from one benchmark — 1536-dimension text embeddings, cosine distance, default search settings — and are a guide, not a guarantee. Other models, dimensions, distance metrics and data distributions behave differently, so measure on your own data before moving a production index to a lower precision. `float32` — on a `float64` column this halves the index. Widely used embedding models emit 32-bit values, so for those nothing is lost; vectors that genuinely carry more than 32 bits of precision will lose some. `float16` — half the memory of `float32`. In that benchmark its results matched `float32` to within 0.1 percentage points. `float8` — a quarter of the memory of `float32`. In that benchmark it scored about 4 percentage points below `float32`, and raising the search effort did not close the gap, so treat the reduction as permanent for a given index. `float64` — accepted only for a column that already holds double-precision values; it cannot add precision the stored data does not have. `int8` — for an `ivf` index only, and its default. A quarter of the size of `float32`, which is a quarter of the bytes every search reads. On the benchmark this index was designed against it found about 99.5% of the neighbours an exact search finds. With `cosine` that accuracy holds however widely your vectors vary in magnitude; with `l2` it falls as they spread — around 93% of the neighbours once the largest magnitude is about 16 times the smallest, and lower beyond that. Use `float32` instead to store the column as written, at four times the size and four times the bytes per search. An `ivf` index accepts `int8` and `float32` only: it stores its copy as a table, and the remaining values have no column type to be stored in or are no smaller than `int8`. An `hnsw` index accepts everything except `int8`; `float8` is its 8-bit option. Changing this means dropping the index and creating it again. It affects only the index: the table's own values are never altered, and text columns indexed with a generated embedding are not re-embedded. | [optional] ## Example diff --git a/docs/IndexEntryResponse.md b/docs/IndexEntryResponse.md index 0646329..c23dd36 100644 --- a/docs/IndexEntryResponse.md +++ b/docs/IndexEntryResponse.md @@ -6,11 +6,13 @@ One index in a cross-table listing: the index itself plus the connection, schema Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- +**algorithm** | **str** | How this vector index organises the vectors it searches: `hnsw` or `ivf`. Absent for BM25 and sorted indexes. | [optional] **columns** | **List[str]** | | **created_at** | **datetime** | | **index_name** | **str** | | **index_type** | **str** | | **metric** | **str** | Distance metric this index was built with. Only present for vector indexes. | [optional] +**probe_fraction** | **float** | How much of an `ivf` index a search reads, as a fraction greater than 0 and at most 1, when it was created with an explicit one. Absent means the server's default. Also absent for every other kind of index. | [optional] **source_column** | **str** | Source text column for an embedding-backed vector index. A query searches it via `vector_distance(<source_column>, …)`; the indexed `columns` hold the generated embedding column instead. Absent for BM25, sorted, and direct (existing-column) vector indexes. | [optional] **status** | [**IndexStatus**](IndexStatus.md) | | **updated_at** | **datetime** | | diff --git a/docs/IndexInfoResponse.md b/docs/IndexInfoResponse.md index f9a7284..e418e5a 100644 --- a/docs/IndexInfoResponse.md +++ b/docs/IndexInfoResponse.md @@ -6,11 +6,13 @@ Result payload for a `create_index` job, and response for index endpoints. Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- +**algorithm** | **str** | How this vector index organises the vectors it searches: `hnsw` or `ivf`. Absent for BM25 and sorted indexes. | [optional] **columns** | **List[str]** | | **created_at** | **datetime** | | **index_name** | **str** | | **index_type** | **str** | | **metric** | **str** | Distance metric this index was built with. Only present for vector indexes. | [optional] +**probe_fraction** | **float** | How much of an `ivf` index a search reads, as a fraction greater than 0 and at most 1, when it was created with an explicit one. Absent means the server's default. Also absent for every other kind of index. | [optional] **source_column** | **str** | Source text column for an embedding-backed vector index. A query searches it via `vector_distance(<source_column>, …)`; the indexed `columns` hold the generated embedding column instead. Absent for BM25, sorted, and direct (existing-column) vector indexes. | [optional] **status** | [**IndexStatus**](IndexStatus.md) | | **updated_at** | **datetime** | | diff --git a/docs/JobResult.md b/docs/JobResult.md index 58f2b81..60cd9bf 100644 --- a/docs/JobResult.md +++ b/docs/JobResult.md @@ -6,11 +6,13 @@ What a finished background job produced. Absent while the job is `pending` or `r Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- +**algorithm** | **str** | How this vector index organises the vectors it searches: `hnsw` or `ivf`. Absent for BM25 and sorted indexes. | [optional] **columns** | **List[str]** | | **created_at** | **datetime** | | **index_name** | **str** | | **index_type** | **str** | | **metric** | **str** | Distance metric this index was built with. Only present for vector indexes. | [optional] +**probe_fraction** | **float** | How much of an `ivf` index a search reads, as a fraction greater than 0 and at most 1, when it was created with an explicit one. Absent means the server's default. Also absent for every other kind of index. | [optional] **source_column** | **str** | Source text column for an embedding-backed vector index. A query searches it via `vector_distance(<source_column>, …)`; the indexed `columns` hold the generated embedding column instead. Absent for BM25, sorted, and direct (existing-column) vector indexes. | [optional] **status** | [**IndexStatus**](IndexStatus.md) | | **updated_at** | **datetime** | | diff --git a/hotdata/models/create_index_request.py b/hotdata/models/create_index_request.py index 03e2d27..8e66c8c 100644 --- a/hotdata/models/create_index_request.py +++ b/hotdata/models/create_index_request.py @@ -19,7 +19,7 @@ import json from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictStr, field_validator -from typing import Any, ClassVar, Dict, List, Optional +from typing import Any, ClassVar, Dict, List, Optional, Union from typing_extensions import Annotated from typing import Optional, Set from typing_extensions import Self @@ -28,6 +28,7 @@ class CreateIndexRequest(BaseModel): """ Request body for POST .../indexes One constraint spans the whole table rather than this request alone: a vector index that generates its own embeddings — that is, one created with `embedding_provider_id` — has to be the only index on its table. So a table that already carries any index (sorted, full-text, or vector) will not accept an embedding-backed vector index, and a table that already carries an embedding-backed vector index will not accept any further index of any type. To move between the two arrangements, drop what is there first. Plan for it when designing a table: combining full-text search with generated embeddings on one table is not possible, so use a separate table for the second index, or supply the embeddings yourself. A vector index over a column that already holds vectors — no `embedding_provider_id` — is not affected and coexists with other indexes normally. Embedding generation also rewrites the table to add its generated column, and that rewrite cannot preserve a declared partition or sort order. An embedding-backed vector index is therefore refused on a table declaring either. """ # noqa: E501 + algorithm: Optional[StrictStr] = Field(default=None, description="How a vector index organises the vectors it searches. Omit this field for `hnsw`, which is the default. `hnsw` — builds a graph of the vectors and keeps it in memory. Searches are very fast, and the memory a search needs grows with the whole table, so a large enough table cannot be served at all. `ivf` — groups the vectors into clusters and reads only the clusters nearest the search. Searches are considerably slower than `hnsw`, and the memory a search needs follows how much of the index it reads rather than the size of the table, so a table far too large for `hnsw` can still be searched. It keeps a copy of the table's rows beside the vectors so a search is answered without reading the table; that copy is extra storage, and how much depends on `vector_precision`, which decides how compactly the copied vectors are held. Available for columns that already hold vectors, with the `l2` and `cosine` metrics.") var_async: Optional[StrictBool] = Field(default=False, description="When true, create the index as a background job and return a job ID for polling.", alias="async") async_after_ms: Optional[Annotated[int, Field(strict=True, ge=1000)]] = Field(default=None, description="If set (requires `async` = true), wait up to this many milliseconds for the index build to finish: if it completes in time the index is returned (201), otherwise a 202 with a job ID to poll. Must be between 1000 and the server maximum; a value out of that range, or set without `async` = true, is rejected with 400.") columns: List[StrictStr] = Field(description="Columns to index. Required for all index types.") @@ -37,9 +38,21 @@ class CreateIndexRequest(BaseModel): index_name: StrictStr index_type: Optional[StrictStr] = Field(default='sorted', description="Index type. `sorted` supports range queries, `bm25` full-text search, and `vector` similarity search.") metric: Optional[StrictStr] = Field(default=None, description="Distance metric for vector indexes: \"l2\", \"cosine\", or \"dot\". When omitted, defaults to \"l2\" for float array columns or the provider's preferred metric for text columns with auto-embedding.") + nlist: Optional[Annotated[int, Field(le=65536, strict=True, ge=1)]] = Field(default=None, description="Number of clusters an `ivf` index divides the vectors into. More clusters means each one holds fewer vectors, so a search of the same effort reads less data. Omit this to let the number be chosen from the table's size.") output_column: Optional[StrictStr] = Field(default=None, description="Custom name for the generated embedding column. Defaults to `{column}_embedding`.") - vector_precision: Optional[StrictStr] = Field(default=None, description="How precisely a vector index stores each number of a vector. Lower precision shrinks the index so a larger table can be indexed within the same memory, and lets searches run on a smaller instance. Omit this field to store vectors at the same precision as the column, which is the default. The quality figures below come from one benchmark — 1536-dimension text embeddings, cosine distance, default search settings — and are a guide, not a guarantee. Other models, dimensions, distance metrics and data distributions behave differently, so measure on your own data before moving a production index to a lower precision. `float32` — on a `float64` column this halves the index. Widely used embedding models emit 32-bit values, so for those nothing is lost; vectors that genuinely carry more than 32 bits of precision will lose some. `float16` — half the memory of `float32`. In that benchmark its results matched `float32` to within 0.1 percentage points. `float8` — a quarter of the memory of `float32`. In that benchmark it scored about 4 percentage points below `float32`, and raising the search effort did not close the gap, so treat the reduction as permanent for a given index. `float64` — accepted only for a column that already holds double-precision values; it cannot add precision the stored data does not have. Changing this means dropping the index and creating it again. It affects only the index: the table's own values are never altered, and text columns indexed with a generated embedding are not re-embedded.") - __properties: ClassVar[List[str]] = ["async", "async_after_ms", "columns", "description", "dimensions", "embedding_provider_id", "index_name", "index_type", "metric", "output_column", "vector_precision"] + probe_fraction: Optional[Union[Annotated[float, Field(le=1, strict=True, ge=0)], Annotated[int, Field(le=1, strict=True, ge=0)]]] = Field(default=None, description="How much of an `ivf` index a search reads, as a fraction greater than 0 and at most 1. Higher finds more of the true nearest neighbours and takes longer. This is a fraction rather than a number of clusters on purpose: the same number of clusters is a different share of the index whenever `nlist` changes, and results would quietly get worse. Omit this for the server's default.") + vector_precision: Optional[StrictStr] = Field(default=None, description="How precisely a vector index stores each number of a vector. Lower precision shrinks the index so a larger table can be indexed within the same memory, and lets searches run on a smaller instance. For an `ivf` index it also shrinks what every search reads, because a search reads part of that stored copy. Omit this field to get each algorithm's own default: an `hnsw` index stores vectors at the same precision as the column, and an `ivf` index stores them as `int8`. The quality figures below come from one benchmark — 1536-dimension text embeddings, cosine distance, default search settings — and are a guide, not a guarantee. Other models, dimensions, distance metrics and data distributions behave differently, so measure on your own data before moving a production index to a lower precision. `float32` — on a `float64` column this halves the index. Widely used embedding models emit 32-bit values, so for those nothing is lost; vectors that genuinely carry more than 32 bits of precision will lose some. `float16` — half the memory of `float32`. In that benchmark its results matched `float32` to within 0.1 percentage points. `float8` — a quarter of the memory of `float32`. In that benchmark it scored about 4 percentage points below `float32`, and raising the search effort did not close the gap, so treat the reduction as permanent for a given index. `float64` — accepted only for a column that already holds double-precision values; it cannot add precision the stored data does not have. `int8` — for an `ivf` index only, and its default. A quarter of the size of `float32`, which is a quarter of the bytes every search reads. On the benchmark this index was designed against it found about 99.5% of the neighbours an exact search finds. With `cosine` that accuracy holds however widely your vectors vary in magnitude; with `l2` it falls as they spread — around 93% of the neighbours once the largest magnitude is about 16 times the smallest, and lower beyond that. Use `float32` instead to store the column as written, at four times the size and four times the bytes per search. An `ivf` index accepts `int8` and `float32` only: it stores its copy as a table, and the remaining values have no column type to be stored in or are no smaller than `int8`. An `hnsw` index accepts everything except `int8`; `float8` is its 8-bit option. Changing this means dropping the index and creating it again. It affects only the index: the table's own values are never altered, and text columns indexed with a generated embedding are not re-embedded.") + __properties: ClassVar[List[str]] = ["algorithm", "async", "async_after_ms", "columns", "description", "dimensions", "embedding_provider_id", "index_name", "index_type", "metric", "nlist", "output_column", "probe_fraction", "vector_precision"] + + @field_validator('algorithm') + def algorithm_validate_enum(cls, value): + """Validates the enum""" + if value is None: + return value + + if value not in set(['hnsw', 'ivf']): + raise ValueError("must be one of enum values ('hnsw', 'ivf')") + return value @field_validator('index_type') def index_type_validate_enum(cls, value): @@ -57,8 +70,8 @@ def vector_precision_validate_enum(cls, value): if value is None: return value - if value not in set(['float64', 'float32', 'float16', 'float8']): - raise ValueError("must be one of enum values ('float64', 'float32', 'float16', 'float8')") + if value not in set(['float64', 'float32', 'float16', 'float8', 'int8']): + raise ValueError("must be one of enum values ('float64', 'float32', 'float16', 'float8', 'int8')") return value model_config = ConfigDict( @@ -125,11 +138,21 @@ def to_dict(self) -> Dict[str, Any]: if self.metric is None and "metric" in self.model_fields_set: _dict['metric'] = None + # set to None if nlist (nullable) is None + # and model_fields_set contains the field + if self.nlist is None and "nlist" in self.model_fields_set: + _dict['nlist'] = None + # set to None if output_column (nullable) is None # and model_fields_set contains the field if self.output_column is None and "output_column" in self.model_fields_set: _dict['output_column'] = None + # set to None if probe_fraction (nullable) is None + # and model_fields_set contains the field + if self.probe_fraction is None and "probe_fraction" in self.model_fields_set: + _dict['probe_fraction'] = None + return _dict @classmethod @@ -142,6 +165,7 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: return cls.model_validate(obj) _obj = cls.model_validate({ + "algorithm": obj.get("algorithm"), "async": obj.get("async") if obj.get("async") is not None else False, "async_after_ms": obj.get("async_after_ms"), "columns": obj.get("columns"), @@ -151,7 +175,9 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: "index_name": obj.get("index_name"), "index_type": obj.get("index_type") if obj.get("index_type") is not None else 'sorted', "metric": obj.get("metric"), + "nlist": obj.get("nlist"), "output_column": obj.get("output_column"), + "probe_fraction": obj.get("probe_fraction"), "vector_precision": obj.get("vector_precision") }) return _obj diff --git a/hotdata/models/index_entry_response.py b/hotdata/models/index_entry_response.py index e4fc533..1cfb039 100644 --- a/hotdata/models/index_entry_response.py +++ b/hotdata/models/index_entry_response.py @@ -19,8 +19,8 @@ import json from datetime import datetime -from pydantic import BaseModel, ConfigDict, Field, StrictStr -from typing import Any, ClassVar, Dict, List, Optional +from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StrictInt, StrictStr +from typing import Any, ClassVar, Dict, List, Optional, Union from hotdata.models.index_status import IndexStatus from typing import Optional, Set from typing_extensions import Self @@ -29,11 +29,13 @@ class IndexEntryResponse(BaseModel): """ One index in a cross-table listing: the index itself plus the connection, schema, and table it belongs to. """ # noqa: E501 + algorithm: Optional[StrictStr] = Field(default=None, description="How this vector index organises the vectors it searches: `hnsw` or `ivf`. Absent for BM25 and sorted indexes.") columns: List[StrictStr] created_at: datetime index_name: StrictStr index_type: StrictStr metric: Optional[StrictStr] = Field(default=None, description="Distance metric this index was built with. Only present for vector indexes.") + probe_fraction: Optional[Union[StrictFloat, StrictInt]] = Field(default=None, description="How much of an `ivf` index a search reads, as a fraction greater than 0 and at most 1, when it was created with an explicit one. Absent means the server's default. Also absent for every other kind of index.") source_column: Optional[StrictStr] = Field(default=None, description="Source text column for an embedding-backed vector index. A query searches it via `vector_distance(, …)`; the indexed `columns` hold the generated embedding column instead. Absent for BM25, sorted, and direct (existing-column) vector indexes.") status: IndexStatus updated_at: datetime @@ -41,7 +43,7 @@ class IndexEntryResponse(BaseModel): connection_id: Optional[StrictStr] = None schema_name: StrictStr table_name: StrictStr - __properties: ClassVar[List[str]] = ["columns", "created_at", "index_name", "index_type", "metric", "source_column", "status", "updated_at", "vector_precision", "connection_id", "schema_name", "table_name"] + __properties: ClassVar[List[str]] = ["algorithm", "columns", "created_at", "index_name", "index_type", "metric", "probe_fraction", "source_column", "status", "updated_at", "vector_precision", "connection_id", "schema_name", "table_name"] model_config = ConfigDict( populate_by_name=True, @@ -99,11 +101,13 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: return cls.model_validate(obj) _obj = cls.model_validate({ + "algorithm": obj.get("algorithm"), "columns": obj.get("columns"), "created_at": obj.get("created_at"), "index_name": obj.get("index_name"), "index_type": obj.get("index_type"), "metric": obj.get("metric"), + "probe_fraction": obj.get("probe_fraction"), "source_column": obj.get("source_column"), "status": obj.get("status"), "updated_at": obj.get("updated_at"), diff --git a/hotdata/models/index_info_response.py b/hotdata/models/index_info_response.py index 817ab17..92db1ed 100644 --- a/hotdata/models/index_info_response.py +++ b/hotdata/models/index_info_response.py @@ -19,8 +19,8 @@ import json from datetime import datetime -from pydantic import BaseModel, ConfigDict, Field, StrictStr -from typing import Any, ClassVar, Dict, List, Optional +from pydantic import BaseModel, ConfigDict, Field, StrictFloat, StrictInt, StrictStr +from typing import Any, ClassVar, Dict, List, Optional, Union from hotdata.models.index_status import IndexStatus from typing import Optional, Set from typing_extensions import Self @@ -29,16 +29,18 @@ class IndexInfoResponse(BaseModel): """ Result payload for a `create_index` job, and response for index endpoints. """ # noqa: E501 + algorithm: Optional[StrictStr] = Field(default=None, description="How this vector index organises the vectors it searches: `hnsw` or `ivf`. Absent for BM25 and sorted indexes.") columns: List[StrictStr] created_at: datetime index_name: StrictStr index_type: StrictStr metric: Optional[StrictStr] = Field(default=None, description="Distance metric this index was built with. Only present for vector indexes.") + probe_fraction: Optional[Union[StrictFloat, StrictInt]] = Field(default=None, description="How much of an `ivf` index a search reads, as a fraction greater than 0 and at most 1, when it was created with an explicit one. Absent means the server's default. Also absent for every other kind of index.") source_column: Optional[StrictStr] = Field(default=None, description="Source text column for an embedding-backed vector index. A query searches it via `vector_distance(, …)`; the indexed `columns` hold the generated embedding column instead. Absent for BM25, sorted, and direct (existing-column) vector indexes.") status: IndexStatus updated_at: datetime vector_precision: Optional[StrictStr] = Field(default=None, description="How precisely this vector index stores each number of a vector, when it was created with an explicit precision. Absent means it stores at the same precision as the column, which is the default. Also absent for BM25 and sorted indexes.") - __properties: ClassVar[List[str]] = ["columns", "created_at", "index_name", "index_type", "metric", "source_column", "status", "updated_at", "vector_precision"] + __properties: ClassVar[List[str]] = ["algorithm", "columns", "created_at", "index_name", "index_type", "metric", "probe_fraction", "source_column", "status", "updated_at", "vector_precision"] model_config = ConfigDict( populate_by_name=True, @@ -79,11 +81,21 @@ def to_dict(self) -> Dict[str, Any]: exclude=excluded_fields, exclude_none=True, ) + # set to None if algorithm (nullable) is None + # and model_fields_set contains the field + if self.algorithm is None and "algorithm" in self.model_fields_set: + _dict['algorithm'] = None + # set to None if metric (nullable) is None # and model_fields_set contains the field if self.metric is None and "metric" in self.model_fields_set: _dict['metric'] = None + # set to None if probe_fraction (nullable) is None + # and model_fields_set contains the field + if self.probe_fraction is None and "probe_fraction" in self.model_fields_set: + _dict['probe_fraction'] = None + # set to None if source_column (nullable) is None # and model_fields_set contains the field if self.source_column is None and "source_column" in self.model_fields_set: @@ -106,11 +118,13 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: return cls.model_validate(obj) _obj = cls.model_validate({ + "algorithm": obj.get("algorithm"), "columns": obj.get("columns"), "created_at": obj.get("created_at"), "index_name": obj.get("index_name"), "index_type": obj.get("index_type"), "metric": obj.get("metric"), + "probe_fraction": obj.get("probe_fraction"), "source_column": obj.get("source_column"), "status": obj.get("status"), "updated_at": obj.get("updated_at"), diff --git a/test/test_create_index_request.py b/test/test_create_index_request.py index 03bbc36..b2ad29e 100644 --- a/test/test_create_index_request.py +++ b/test/test_create_index_request.py @@ -36,6 +36,7 @@ def make_instance(self, include_optional) -> CreateIndexRequest: model = CreateIndexRequest() if include_optional: return CreateIndexRequest( + algorithm = 'hnsw', var_async = True, async_after_ms = 1000, columns = [customer_id], @@ -45,7 +46,9 @@ def make_instance(self, include_optional) -> CreateIndexRequest: index_name = 'orders_customer_id', index_type = 'sorted', metric = '', + nlist = 1, output_column = '', + probe_fraction = 0, vector_precision = 'float64' ) else: diff --git a/test/test_index_entry_response.py b/test/test_index_entry_response.py index d3b6835..e3a6594 100644 --- a/test/test_index_entry_response.py +++ b/test/test_index_entry_response.py @@ -36,6 +36,7 @@ def make_instance(self, include_optional) -> IndexEntryResponse: model = IndexEntryResponse() if include_optional: return IndexEntryResponse( + algorithm = '', columns = [ '' ], @@ -43,6 +44,7 @@ def make_instance(self, include_optional) -> IndexEntryResponse: index_name = '', index_type = '', metric = '', + probe_fraction = 1.337, source_column = '', status = 'ready', updated_at = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), diff --git a/test/test_index_info_response.py b/test/test_index_info_response.py index c0dd6c4..c4b0e66 100644 --- a/test/test_index_info_response.py +++ b/test/test_index_info_response.py @@ -36,6 +36,7 @@ def make_instance(self, include_optional) -> IndexInfoResponse: model = IndexInfoResponse() if include_optional: return IndexInfoResponse( + algorithm = '', columns = [ '' ], @@ -43,6 +44,7 @@ def make_instance(self, include_optional) -> IndexInfoResponse: index_name = '', index_type = '', metric = '', + probe_fraction = 1.337, source_column = '', status = 'ready', updated_at = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), diff --git a/test/test_job_result.py b/test/test_job_result.py index aed99c4..6802313 100644 --- a/test/test_job_result.py +++ b/test/test_job_result.py @@ -36,6 +36,7 @@ def make_instance(self, include_optional) -> JobResult: model = JobResult() if include_optional: return JobResult( + algorithm = '', columns = [ '' ], @@ -43,6 +44,7 @@ def make_instance(self, include_optional) -> JobResult: index_name = '', index_type = '', metric = '', + probe_fraction = 1.337, source_column = '', status = 'ready', updated_at = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), diff --git a/test/test_list_indexes_response.py b/test/test_list_indexes_response.py index c332be9..fdd3ed3 100644 --- a/test/test_list_indexes_response.py +++ b/test/test_list_indexes_response.py @@ -38,6 +38,7 @@ def make_instance(self, include_optional) -> ListIndexesResponse: return ListIndexesResponse( indexes = [ hotdata.models.index_info_response.IndexInfoResponse( + algorithm = '', columns = [ '' ], @@ -45,6 +46,7 @@ def make_instance(self, include_optional) -> ListIndexesResponse: index_name = '', index_type = '', metric = '', + probe_fraction = 1.337, source_column = '', status = 'ready', updated_at = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), @@ -55,6 +57,7 @@ def make_instance(self, include_optional) -> ListIndexesResponse: return ListIndexesResponse( indexes = [ hotdata.models.index_info_response.IndexInfoResponse( + algorithm = '', columns = [ '' ], @@ -62,6 +65,7 @@ def make_instance(self, include_optional) -> ListIndexesResponse: index_name = '', index_type = '', metric = '', + probe_fraction = 1.337, source_column = '', status = 'ready', updated_at = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'),