diff --git a/.github/workflows/upgrade_dependencies.yml b/.github/workflows/upgrade_dependencies.yml index bdc98691..fef5cb45 100644 --- a/.github/workflows/upgrade_dependencies.yml +++ b/.github/workflows/upgrade_dependencies.yml @@ -11,7 +11,7 @@ jobs: matrix: os: ['ubuntu-latest'] package: ["mp-api"] - python-version: ["3.11", "3.12", "3.13", "3.14"] + python-version: ["3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 with: diff --git a/README.md b/README.md index 9de791ed..1044e0f9 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,6 @@ [![testing](https://github.com/materialsproject/api/actions/workflows/testing.yml/badge.svg?branch=main)](https://github.com/materialsproject/api/actions?query=workflow%3Atesting+branch%3Amain) [![codecov](https://codecov.io/gh/materialsproject/api/branch/main/graph/badge.svg)](https://codecov.io/gh/materialsproject/api) -![python](https://img.shields.io/badge/Python-3.11+-blue.svg?logo=python&logoColor=white) +![python](https://img.shields.io/badge/Python-3.12+-blue.svg?logo=python&logoColor=white) This repository is the development environment for the new Materials Project API. A core client implementation will reside here. For information on how to use the API, please see the updated [documentation](https://docs.materialsproject.org/downloading-data/how-do-i-download-the-materials-project-database). diff --git a/mp_api/_test_utils.py b/mp_api/_test_utils.py index c11b0ea0..332eaef7 100644 --- a/mp_api/_test_utils.py +++ b/mp_api/_test_utils.py @@ -139,7 +139,6 @@ def _normalize(doc, field: str): if k not in ("_page", "_sort_fields", "chunk_size", "fields") } for sort_field in [sort_fields] if isinstance(sort_fields, str) else sort_fields: - asc = search_method( _page=1, _sort_fields=sort_field, diff --git a/mp_api/client/__init__.py b/mp_api/client/__init__.py index 3eb2b879..75307bf6 100644 --- a/mp_api/client/__init__.py +++ b/mp_api/client/__init__.py @@ -16,4 +16,5 @@ except PackageNotFoundError: # pragma: no cover __version__ = os.getenv("SETUPTOOLS_SCM_PRETEND_VERSION", "") -logging.getLogger(__name__).addHandler(logging.NullHandler()) +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) diff --git a/mp_api/client/core/client.py b/mp_api/client/core/client.py index 647f32ec..38853eb3 100644 --- a/mp_api/client/core/client.py +++ b/mp_api/client/core/client.py @@ -43,6 +43,7 @@ from urllib3.util.retry import Retry from mp_api.client._server_utils import get_consumer, get_user_api_key, is_dev_env +from mp_api.client.core.delta import DeltaCatalog from mp_api.client.core.exceptions import ( MPRestError, MPRestWarning, @@ -61,6 +62,8 @@ from collections.abc import Callable, Iterable, Iterator from typing import Any + from arro3.core import RecordBatchReader + from mp_api.client.core.utils import LazyImport try: @@ -88,13 +91,7 @@ "thermo", ] -hdlr = logging.StreamHandler() -fmt = logging.Formatter("%(name)s - %(levelname)s - %(message)s") -hdlr.setFormatter(fmt) - logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) -logger.addHandler(hdlr) def _batched(iterable: Iterable, n: int) -> Iterator: @@ -106,28 +103,42 @@ def _batched(iterable: Iterable, n: int) -> Iterator: class QueryBuilderWithCache(QueryBuilder): + def __init__(self, catalog: DeltaCatalog | None = None, _warn: bool = True) -> None: + """Deprecated: use `mp_api.client.core.delta.DeltaCatalog`. - def __init__(self) -> None: - """Extend deltalake.QueryBuilder with stored DeltaTables. - - The deltalake.QueryBuilder class does not permit introspection - of registered DeltaTables through the python API. - - Re-registering a DeltaTable - (1) wastes time by reading its metadata - (2) raises an exception because a table is already registered + Kept for backwards compatibility. Tables registered here, and + queries run through it, are delegated to a `DeltaCatalog`. Resters + given this object via `query_builder=` share its catalog. - This class simply allows for caching the DeltaTable instances - and table names on the QueryBuilder class. + Args: + catalog (DeltaCatalog or None) : catalog to delegate to. + A new one is created if None. + _warn (bool) : internal, whether to emit a DeprecationWarning """ - # Dict of table names (labels) to DeltaTable instances - self._delta_tables: dict[str, DeltaTable] = {} + if _warn: + warnings.warn( + "QueryBuilderWithCache is deprecated and will be removed in a future " + "release. Pass `delta_catalog=DeltaCatalog()` " + "(from mp_api.client.core.delta) to MPRester instead.", + category=DeprecationWarning, + stacklevel=2, + ) + self.catalog: DeltaCatalog = catalog if catalog is not None else DeltaCatalog() super().__init__() + @property + def _delta_tables(self) -> dict[str, DeltaTable]: + """Map of table names (labels) to DeltaTable instances.""" + return self.catalog.tables + def register(self, table_name: str, delta_table: DeltaTable) -> QueryBuilder: - """Register and cache a DeltaTable.""" - self._delta_tables[table_name] = delta_table - return super().register(table_name, delta_table) + """Register a DeltaTable in the underlying catalog.""" + self.catalog.add(table_name, delta_table) + return self + + def execute(self, sql: str) -> RecordBatchReader: + """Execute SQL against the tables in the underlying catalog.""" + return self.catalog.execute_stream(sql) class _Rester: @@ -148,6 +159,7 @@ def __init__( ) = MAPI_CLIENT_SETTINGS.LOCAL_DATASET_CACHE, force_renew: bool = False, query_builder: QueryBuilderWithCache | None = None, + delta_catalog: DeltaCatalog | None = None, **kwargs, ) -> None: """Initialize a RESTer. @@ -182,8 +194,12 @@ def __init__( local_dataset_cache: Target directory for downloading full datasets. Defaults to 'mp_datasets' in the user's home directory force_renew: Option to overwrite existing local dataset - query_builder : Instance of QueryBuilderWithCache to use in querying delta tables + query_builder : DEPRECATED, use `delta_catalog`. Instance of QueryBuilderWithCache + whose catalog is used for querying delta tables. NOTE: Must be a QueryBuilderWithCache, a deltalake.QueryBuilder will be ignored. + delta_catalog : Instance of DeltaCatalog to use for querying delta tables. + Share one instance across resters (e.g. one per web-server worker) to + reuse loaded table snapshots. Takes precedence over `query_builder`. **kwargs: access to legacy kwargs that may be in the process of being deprecated """ self.api_key = get_user_api_key(api_key=api_key) @@ -210,6 +226,9 @@ def __init__( self._query_builder = ( query_builder if isinstance(query_builder, QueryBuilderWithCache) else None ) + if self._query_builder is not None and delta_catalog is None: + delta_catalog = self._query_builder.catalog + self._delta_catalog: DeltaCatalog | None = delta_catalog if "monty_decode" in kwargs: # Pop to not repeatedly trigger warning to the user @@ -230,9 +249,24 @@ def session(self) -> requests.Session: return self._session @property - def query_builder(self): - if not self._query_builder: - self._query_builder = QueryBuilderWithCache() + def delta_catalog(self) -> DeltaCatalog: + """The DeltaCatalog used for delta-backed queries, created on first use.""" + if self._delta_catalog is None: + self._delta_catalog = DeltaCatalog() + return self._delta_catalog + + @property + def query_builder(self) -> QueryBuilderWithCache: + """Deprecated: use `delta_catalog`.""" + warnings.warn( + "`query_builder` is deprecated, use `delta_catalog` instead.", + category=DeprecationWarning, + stacklevel=2, + ) + if self._query_builder is None: + self._query_builder = QueryBuilderWithCache( + catalog=self.delta_catalog, _warn=False + ) return self._query_builder @staticmethod @@ -336,6 +370,7 @@ def __init__( ) = MAPI_CLIENT_SETTINGS.LOCAL_DATASET_CACHE, force_renew: bool = False, query_builder: QueryBuilderWithCache | None = None, + delta_catalog: DeltaCatalog | None = None, s3_client: Any | None = None, timeout: int = 20, **kwargs, @@ -375,9 +410,11 @@ def __init__( local_dataset_cache: Target directory for downloading full datasets. Defaults to 'mp_datasets' in the user's home directory force_renew: Option to overwrite existing local dataset - query_builder : Instance of QueryBuilderWithCache to use in querying delta tables + query_builder : DEPRECATED, use `delta_catalog`. Instance of QueryBuilderWithCache + whose catalog is used for querying delta tables. NOTE: Must be a QueryBuilderWithCache, a deltalake.QueryBuilder will be ignored. - s3_client: boto3 S3 client object with which to connect to the object stores.ct to the object stores.ct to the object stores. + delta_catalog : Instance of DeltaCatalog to use for querying delta tables. + s3_client: boto3 S3 client object with which to connect to the object stores. timeout: Time in seconds to wait until a request timeout error is thrown **kwargs: access to legacy kwargs that may be in the process of being deprecated """ @@ -393,6 +430,7 @@ def __init__( local_dataset_cache=local_dataset_cache, force_renew=force_renew, query_builder=query_builder, + delta_catalog=delta_catalog, **kwargs, ) @@ -594,22 +632,24 @@ def _get_delta_table( prefix: str, connector: str = "s3a", label: str | None = None, + refresh: bool = False, ) -> tuple[str, DeltaTable]: """Either create a new DeltaTable, or retrieve a cached one. - If creating a new DeltaTable, will also register in self.query_builder + If creating a new DeltaTable, will also register it in self.delta_catalog Args: bucket (str) : name of the bucket in S3 prefix (str) : name of the prefix in S3 connector (str) : s3, s3n, s3a (default), or other valid Hadoop connector string. - label (str or None) : optional label for the table in the - cached query builder - If `None`, will be gleaned from the URI + label (str or None) : optional label (SQL table name) for the + table in the catalog. If `None`, will be gleaned from the URI + refresh (bool) : if the table is already cached, reload its + snapshot to the latest version first Returns: - str : the table name in the stored query builder + str : the table name in the catalog DeltaTable : If one exists at the specified bucket / prefix, will retrieve the cached instance. """ @@ -621,31 +661,22 @@ def _get_delta_table( if not uri.endswith("/"): uri += "/" - try: - stored_label, delta_table = next( - (_label, _table) - for _label, _table in self.query_builder._delta_tables.items() - if _table.table_uri == uri - ) - except StopIteration: - stored_label = None - - if stored_label is None: - delta_table = DeltaTable( - uri, - storage_options={ - "AWS_SKIP_SIGNATURE": "true", - "AWS_REGION": "us-east-1", - "timeout": delta_timeout, - "connect_timeout": delta_timeout, - "pool_idle_timeout": delta_timeout, - "retry_delay": "3", - "max_retries": f"{MAPI_CLIENT_SETTINGS.MAX_RETRIES}", - }, - ) - self.query_builder.register(qb_label, delta_table) + stored_label, delta_table = self.delta_catalog.get_table( + uri, + qb_label, + storage_options={ + "AWS_SKIP_SIGNATURE": "true", + "AWS_REGION": "us-east-1", + "timeout": delta_timeout, + "connect_timeout": delta_timeout, + "pool_idle_timeout": delta_timeout, + "retry_delay": "3", + "max_retries": f"{MAPI_CLIENT_SETTINGS.MAX_RETRIES}", + }, + refresh=refresh, + ) - elif stored_label != qb_label: + if stored_label != qb_label: warnings.warn( f"DeltaTable with URI {uri} already found with different label: " f"Stored label = {stored_label}; submitted label {qb_label}. " @@ -653,13 +684,16 @@ def _get_delta_table( category=MPRestWarning, stacklevel=2, ) - return stored_label, delta_table - return qb_label, delta_table + return stored_label, delta_table - def _query_delta_single(self, query: str) -> pa.Table: + def _query_delta_single(self, query: str, label: str | None = None) -> pa.Table: """Execute a SQL query against a registered Delta table. + If `label` is given and the query fails because a file in the cached + snapshot no longer exists (e.g. the remote table was vacuumed), only + that table is reloaded and the query is retried once. + Wraps the query execution in a try/except to provide a more actionable error message when the underlying Delta query engine fails (e.g., due to network timeouts, missing tables, or @@ -668,6 +702,8 @@ def _query_delta_single(self, query: str) -> pa.Table: Args: query (str): A SQL query string compatible with the QueryBuilder engine. + label (str or None): The registered table the query reads from, + as returned by `_get_delta_table`. Required for retries. Returns: pa.Table: The query result as a PyArrow Table. @@ -679,13 +715,20 @@ def _query_delta_single(self, query: str) -> pa.Table: the underlying cause. """ try: - return pa.table(self.query_builder.execute(query).read_all()) + return self.delta_catalog.execute(query, label=label) except Exception as e: - raise MPRestError( - f"Failed to retrieve object due to: {e}. " - f"If this is a timeout error, try increasing the 'timeout' " - f"parameter on MPRester (current value: {self.timeout}s)." - ) from e + refreshed = any( + "after refreshing" in note for note in getattr(e, "__notes__", []) + ) + hint = ( + f"The DeltaTable '{label}' was refreshed and the query retried once." + if refreshed + else ( + "If this is a timeout error, try increasing the 'timeout' " + f"parameter on MPRester (current value: {self.timeout}s)." + ) + ) + raise MPRestError(f"Failed to retrieve object due to: {e}. {hint}") from e def _query_delta_backed( self, @@ -764,7 +807,8 @@ def _query_delta_backed( ) } - tbl_lbl, tbl = self._get_delta_table(bucket, prefix, label=label) + # Full downloads are one-off, always start from the latest snapshot + tbl_lbl, tbl = self._get_delta_table(bucket, prefix, label=label, refresh=True) controlled_batch_str = ",".join( [f"'{tag}'" for tag in self.access_controlled_batch_ids] @@ -810,7 +854,9 @@ def _query_delta_backed( else None ) - iterator = self.query_builder.execute(f"SELECT * FROM {tbl_lbl} {predicate}") + iterator = self.delta_catalog.execute_stream( + f"SELECT * FROM {tbl_lbl} {predicate}" + ) file_options = ds.ParquetFileFormat().make_write_options(compression="zstd") @@ -1749,7 +1795,7 @@ def __getattr__(self, v: str): db_version=self.db_version, local_dataset_cache=self.local_dataset_cache, force_renew=self.force_renew, - query_builder=self._query_builder, + delta_catalog=self.delta_catalog, ) return self.sub_resters[v] raise AttributeError(f"{self.__class__} has no attribute {v}") diff --git a/mp_api/client/core/delta.py b/mp_api/client/core/delta.py new file mode 100644 index 00000000..69428590 --- /dev/null +++ b/mp_api/client/core/delta.py @@ -0,0 +1,390 @@ +"""Manage DeltaTable snapshots and SQL execution for delta-backed routes. + +`deltalake.QueryBuilder.register` copies a table's snapshot (its list of +data files) into a DataFusion session at registration time. The snapshot +then can't be updated or deregistered, and registering the same name again +raises. + +If the remote table is vacuumed after an update, an old snapshot can point +to data files that no longer exist. The only way to recover is to load the +table again and register it on a new QueryBuilder. + +`DeltaCatalog` owns the DeltaTables and the QueryBuilder they are registered +on. When a query fails because an object is missing, it reloads only the +table named in that query, swaps in a new QueryBuilder with every table +registered on it, and retries the query once. + +Blocking native calls can be sent to a pluggable runner (e.g. a threadpool), +see `DeltaCatalog`. +""" + +from __future__ import annotations + +import logging +import re +import threading +from dataclasses import dataclass +from functools import partial +from typing import TYPE_CHECKING, Any + +import pyarrow as pa +from deltalake import DeltaTable, QueryBuilder + +if TYPE_CHECKING: + from collections.abc import Callable + + from arro3.core import RecordBatchReader + + # Takes a zero-argument function, runs it, returns its result + Runner = Callable[[Callable[[], Any]], Any] + +logger = logging.getLogger(__name__) + + +def _call_directly(fn: Callable[[], Any]) -> Any: + """Default runner: run `fn` in the calling thread.""" + return fn() + + +# Messages that mean a data or log file referenced by the current snapshot +# is gone (e.g. removed by a vacuum on the remote table). +# DataFusion errors reach Python as a generic `DeltaError`, so the message +# has to be matched as well as the exception type. Example: +# "Failed to fetch metadata for file ...: Parquet error: External: +# Object at location ... not found: No such file or directory (os error 2)" +_MISSING_OBJECT_PATTERN = re.compile( + r"object at location .* not found" + r"|notfound" + r"|not found" + r"|no such file or directory" + r"|status code: 404", + re.IGNORECASE, +) + + +def _is_missing_object(exc: BaseException) -> bool: + """Whether an exception means a file in the snapshot no longer exists. + + Args: + exc (BaseException) : exception raised by deltalake / DataFusion + + Returns: + bool + """ + if isinstance(exc, FileNotFoundError): + return True + return bool(_MISSING_OBJECT_PATTERN.search(str(exc))) + + +@dataclass +class _Entry: + """A DeltaTable registered under a label, plus what's needed to reload it.""" + + label: str + uri: str + table: DeltaTable + storage_options: dict[str, str] | None = None + # Increases each time this table is reloaded. Used to tell whether + # another thread already refreshed it. + generation: int = 0 + + +class DeltaCatalog: + """Thread-safe cache of DeltaTables and the QueryBuilder they are registered on. + + One catalog can be shared by every rester in a process. + Snapshots stay fixed until a query fails because a + referenced object is missing, then only that table is reloaded. + + Blocking native work (loading a table's snapshot from storage, and + running a query) can be sent to a `runner`. A runner is any callable + that takes a zero-argument function, runs it, and returns its result + or raises its exception. Under gevent, for example, running native calls + in a real thread stops them from blocking the event loop: + + ```python + import gevent + + catalog = DeltaCatalog(runner=gevent.get_hub().threadpool.apply) + ``` + + or, with a standard executor: + + ```python + from concurrent.futures import ThreadPoolExecutor + + pool = ThreadPoolExecutor(8) + catalog = DeltaCatalog(runner=lambda fn: pool.submit(fn).result()) + ``` + + The catalog's own locking and bookkeeping always run in the calling + thread / greenlet, never inside the runner. The catalog-wide lock is + never held during a load or query, so a slow load of one table doesn't + block queries on other tables. + """ + + def __init__(self, runner: Runner | None = None) -> None: + """Create an empty catalog. + + Args: + runner (callable or None) : runs blocking native calls, see the + class docstring. If None, they run directly in the caller. + """ + self._runner: Runner = runner or _call_directly + self._entries: dict[str, _Entry] = {} # keyed by URI + self._labels: dict[str, str] = {} # label -> URI + self._qb: QueryBuilder = QueryBuilder() + # Guards _entries, _labels, _qb and _load_locks. Only held for + # in-memory work, never across a runner call. + self._lock = threading.RLock() + # One lock per URI, held while that table is (re)loaded so concurrent + # loads of the same table collapse into one. + self._load_locks: dict[str, threading.Lock] = {} + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(tables={self.labels})" + + @property + def runner(self) -> Runner: + """The callable used to run blocking native calls.""" + return self._runner + + @property + def labels(self) -> list[str]: + """Labels (SQL table names) of all registered tables.""" + with self._lock: + return list(self._labels) + + @property + def tables(self) -> dict[str, DeltaTable]: + """Map of label to currently registered DeltaTable.""" + with self._lock: + return { + label: self._entries[uri].table for label, uri in self._labels.items() + } + + def __contains__(self, label: str) -> bool: + with self._lock: + return label in self._labels + + def __len__(self) -> int: + with self._lock: + return len(self._entries) + + def add( + self, + label: str, + table: DeltaTable, + storage_options: dict[str, str] | None = None, + uri: str | None = None, + ) -> tuple[str, DeltaTable]: + """Register an already-constructed DeltaTable. + + If a table with the same URI is already registered, the stored + label and table are returned and `table` is discarded. + + Args: + label (str) : SQL table name + table (DeltaTable) : the table to register + storage_options (dict or None) : storage options used to reload + the table. Defaults to the options the table was created with. + uri (str or None) : cache key. Defaults to `table.table_uri`. + + Returns: + tuple of the label the table is registered under and the DeltaTable. + + Raises: + ValueError: if `label` is already used by a table with a different URI. + """ + uri = uri or table.table_uri + with self._lock: + if (existing := self._entries.get(uri)) is not None: + return existing.label, existing.table + self._check_label_free(label, uri) + self._qb.register(label, table) + self._entries[uri] = _Entry( + label=label, + uri=uri, + table=table, + storage_options=( + storage_options + if storage_options is not None + else getattr(table, "_storage_options", None) + ), + ) + self._labels[label] = uri + return label, table + + def get_table( + self, + uri: str, + label: str, + storage_options: dict[str, str] | None = None, + refresh: bool = False, + ) -> tuple[str, DeltaTable]: + """Get a cached DeltaTable, or load and register it. + + Args: + uri (str) : table URI, also the cache key + label (str) : SQL table name to register a new table under + storage_options (dict or None) : storage options for a newly loaded table + refresh (bool) : if the table is already cached, reload it first. + Skipped if another caller reloaded it while this one waited. + + Returns: + tuple of the label the table is registered under (may differ from + `label` if the URI was registered earlier under another name) and + the DeltaTable. + + Raises: + ValueError: if `label` is already used by a table with a different URI. + """ + with self._lock: + entry = self._entries.get(uri) + if entry is not None and not refresh: + return entry.label, entry.table + if entry is None: + # fail fast, before any network I/O + self._check_label_free(label, uri) + seen_generation = entry.generation if entry is not None else None + load_lock = self._load_lock(uri) + + with load_lock: + with self._lock: + entry = self._entries.get(uri) + + if entry is None: + table = self._load(uri, storage_options) + return self.add(label, table, storage_options=storage_options, uri=uri) + + # Reload only if nobody else loaded / reloaded it while we waited + if seen_generation is not None and entry.generation == seen_generation: + self._reload(entry) + return entry.label, entry.table + + def execute(self, sql: str, label: str | None = None) -> pa.Table: + """Run a SQL query and return the full result. + + If `label` is given and the query fails because an object is missing, + that table is reloaded and the query is retried once. Without + `label`, errors are raised as-is. + + Args: + sql (str) : SQL query + label (str or None) : the table the query reads from + + Returns: + pyarrow.Table + + Raises: + Whatever deltalake / DataFusion / pyarrow raises. If the retry + also fails, its exception has a note saying the table was refreshed. + """ + with self._lock: + qb = self._qb + entry = self._entries.get(self._labels.get(label, "")) if label else None + seen_generation = entry.generation if entry else None + + try: + return self._runner(partial(self._run, qb, sql)) + except Exception as exc: + if entry is None or not _is_missing_object(exc): + raise + logger.warning( + f"Query on DeltaTable '{label}' referenced a missing object, " + f"refreshing snapshot and retrying: {exc}" + ) + + qb = self._refresh(entry, seen_generation) # type: ignore[arg-type] + try: + return self._runner(partial(self._run, qb, sql)) + except Exception as exc: + exc.add_note( + f"Query failed again after refreshing DeltaTable '{label}' ({entry.uri})." + ) + raise + + def execute_stream(self, sql: str) -> RecordBatchReader: + """Run a SQL query and stream the results, without retrying. + + Used for full-dataset downloads, where batches may already have been + written to disk before an error, so a transparent retry isn't safe. + Call `get_table(..., refresh=True)` first to start from the latest snapshot. + + Does not use the runner: batches are fetched lazily as the returned + reader is consumed, in the consuming thread. + + Args: + sql (str) : SQL query + + Returns: + arro3 RecordBatchReader + """ + with self._lock: + qb = self._qb + return qb.execute(sql) + + @staticmethod + def _run(qb: QueryBuilder, sql: str) -> pa.Table: + # Read via pyarrow's C stream interface rather than arro3's + # `read_all()`: arro3 holds the GIL while it fetches every batch, + # pyarrow releases it, so other threads (and, with a threadpool + # runner, the gevent loop) keep running during the fetch. + # A missing-file error can surface while planning (execute) or while + # reading batches; both happen inside the caller's try. + return pa.table(qb.execute(sql)) + + def _load(self, uri: str, storage_options: dict[str, str] | None) -> DeltaTable: + """Load a table snapshot from storage via the runner. Caller holds the URI's load lock.""" + return self._runner(partial(DeltaTable, uri, storage_options=storage_options)) + + def _load_lock(self, uri: str) -> threading.Lock: + """Get or create the load lock for a URI. Caller holds the catalog lock.""" + if (lock := self._load_locks.get(uri)) is None: + lock = self._load_locks[uri] = threading.Lock() + return lock + + def _check_label_free(self, label: str, uri: str) -> None: + if (other := self._labels.get(label)) is not None and other != uri: + raise ValueError( + f"Label '{label}' is already registered for DeltaTable {other}, " + f"cannot register it for {uri}." + ) + + def _refresh(self, entry: _Entry, seen_generation: int) -> QueryBuilder: + """Reload `entry` unless another caller already did, and return the current QueryBuilder.""" + with self._lock: + load_lock = self._load_lock(entry.uri) + with load_lock: + if entry.generation == seen_generation: + self._reload(entry) + with self._lock: + return self._qb + + def _reload(self, entry: _Entry) -> None: + """Reload one table's snapshot and swap in a new QueryBuilder. + + Caller holds the entry's load lock, but not the catalog lock: the + load runs without it, then the catalog lock is taken briefly to swap. + + Builds a new DeltaTable instead of calling `update_incremental()`, in + case the table was rebuilt rather than appended to. Other tables keep + their snapshots; registering them again only copies in-memory state. + Queries already running keep using the old QueryBuilder. + """ + table = self._load(entry.uri, entry.storage_options) + + with self._lock: + # Rebuild from the entries as they are now, so a concurrent + # reload of a different table isn't lost. + qb = QueryBuilder() + for other in self._entries.values(): + qb.register(other.label, table if other is entry else other.table) + entry.table = table + entry.generation += 1 + self._qb = qb + + logger.info( + f"Refreshed DeltaTable '{entry.label}' ({entry.uri}) " + f"to version {table.version()}." + ) diff --git a/mp_api/client/core/schemas.py b/mp_api/client/core/schemas.py index d721ef69..9a886b28 100644 --- a/mp_api/client/core/schemas.py +++ b/mp_api/client/core/schemas.py @@ -2,13 +2,20 @@ from __future__ import annotations -from functools import cached_property +import logging +from functools import cached_property, lru_cache from importlib import import_module +from importlib.metadata import version from itertools import chain from typing import TYPE_CHECKING, ForwardRef, get_args from emmet.core.utils import jsanitize -from pydantic import BaseModel, create_model +from pydantic import ( + BaseModel, + ValidationError, + create_model, + field_validator, +) if TYPE_CHECKING: from collections.abc import Iterator @@ -16,6 +23,13 @@ from pydantic.fields import FieldInfo +logger = logging.getLogger(__name__) + + +@lru_cache(20) +def _warn_failed_field_validation(logger: logging.Logger, msg: str): + logger.warning(msg) + class _DictLikeAccess(BaseModel): """Define a pydantic mix-in which permits dict-like access to model fields.""" @@ -47,7 +61,7 @@ def __str__(self) -> str: return ( f"{self.__class__.__name__}(\n" + "\n".join( - f" {k} ({annos[k]}) : {getattr(self,k)}" for k in populated_fields + f" {k} ({annos[k]}) : {getattr(self, k)}" for k in populated_fields ) + "\n)" ) @@ -56,6 +70,24 @@ def __repr__(self) -> str: """Match output of str().""" return self.__str__() + @field_validator("*", mode="wrap") + @classmethod + def ignore_invalid(cls, value, default_validator, info) -> Any: + try: + return default_validator(value) + except ValidationError: + emmet_version = version("emmet-core") + annotation = cls.model_fields[info.field_name].annotation + _warn_failed_field_validation( + logger, + f"Failed validation on field: '{info.field_name}', received type: '{type(value)}', expected type: '{annotation}'. " + "Field value set to 'None', re-run query with 'document_model=False' to skip validation. " + f"Local 'emmet-core' version is: '{emmet_version}', run 'mpr = MPRester(); mpr.get_emmet_version(mpr.endpoint)' to " + "check the live API server for schema version mismatches and upgrade 'emmet-core' if needed.", + ) + + return None + def _generate_returned_model( doc: dict[str, Any], diff --git a/mp_api/client/mprester.py b/mp_api/client/mprester.py index ebb191bc..ced3f9c7 100644 --- a/mp_api/client/mprester.py +++ b/mp_api/client/mprester.py @@ -64,6 +64,7 @@ from pymatgen.util.typing import SpeciesLike from mp_api.client.core.client import QueryBuilderWithCache + from mp_api.client.core.delta import DeltaCatalog from mp_api.client.core.schemas import _DictLikeAccess @@ -120,6 +121,7 @@ def __init__( ) = MAPI_CLIENT_SETTINGS.LOCAL_DATASET_CACHE, force_renew: bool = False, query_builder: QueryBuilderWithCache | None = None, + delta_catalog: DeltaCatalog | None = None, notify_db_version: bool = False, **kwargs, ): @@ -154,8 +156,13 @@ def __init__( local_dataset_cache: Target directory for downloading full datasets. Defaults to "mp_datasets" in the user's home directory force_renew: Option to overwrite existing local dataset - query_builder : Instance of QueryBuilderWithCache to use in querying delta tables + query_builder : DEPRECATED, use `delta_catalog`. Instance of QueryBuilderWithCache + whose catalog is used for querying delta tables. NOTE: Must be a QueryBuilderWithCache, a deltalake.QueryBuilder will be ignored. + delta_catalog : Instance of DeltaCatalog to use for querying delta tables. + Share one instance across MPRester instances (e.g. one per web-server + worker) to reuse loaded table snapshots. If None, one is created and + shared by all resters under this MPRester. notify_db_version (bool): If True, the current MP database version will be retrieved and logged locally in the ~/.mprester.log.yaml. If the database version changes, you will be notified. The current database version is @@ -177,6 +184,7 @@ def __init__( local_dataset_cache=local_dataset_cache, force_renew=force_renew, query_builder=query_builder, + delta_catalog=delta_catalog, **kwargs, ) @@ -269,7 +277,7 @@ def __init__( db_version=self.db_version, local_dataset_cache=self.local_dataset_cache, force_renew=self.force_renew, - query_builder=self._query_builder, + delta_catalog=self.delta_catalog, ), ) diff --git a/mp_api/client/routes/materials/electronic_structure.py b/mp_api/client/routes/materials/electronic_structure.py index ea42b99f..1c009408 100644 --- a/mp_api/client/routes/materials/electronic_structure.py +++ b/mp_api/client/routes/materials/electronic_structure.py @@ -164,6 +164,7 @@ def es_rester(self) -> ElectronicStructureRester: use_document_model=self.use_document_model, headers=self.headers, mute_progress_bars=self.mute_progress_bars, + delta_catalog=self.delta_catalog, ) return self._es_rester @@ -298,7 +299,7 @@ def get_bandstructure_from_task_id( if path_type: query += f"\nAND path_convention='{path_type}'" - table = self._query_delta_single(query) + table = self._query_delta_single(query, label=bs_lbl) if len(deser := table.to_pylist(maps_as_pydicts="strict")) > 0: if load_projections: proj_bs_label, _ = self._get_delta_table( @@ -307,7 +308,7 @@ def get_bandstructure_from_task_id( label="bandstructure_projections", ) proj_table = self._query_delta_single( - query.replace(bs_lbl, proj_bs_label) + query.replace(bs_lbl, proj_bs_label), label=proj_bs_label ) if ( len(deser_proj := proj_table.to_pylist(maps_as_pydicts="strict")) @@ -553,7 +554,7 @@ def get_dos_from_task_id( rt = RunType(run_type) if isinstance(run_type, str) else run_type query += f"\nAND run_type='{rt.value}'" - table = self._query_delta_single(query) + table = self._query_delta_single(query, label=dos_lbl) if len(deser := table.to_pylist(maps_as_pydicts="strict")) > 0: if load_projections: proj_dos_label, _ = self._get_delta_table( @@ -562,7 +563,7 @@ def get_dos_from_task_id( label="dos_projections", ) proj_table = self._query_delta_single( - query.replace(dos_lbl, proj_dos_label) + query.replace(dos_lbl, proj_dos_label), label=proj_dos_label ) if ( len(deser_proj := proj_table.to_pylist(maps_as_pydicts="strict")) diff --git a/mp_api/client/routes/materials/phonon.py b/mp_api/client/routes/materials/phonon.py index 4dc778a1..81f39122 100644 --- a/mp_api/client/routes/materials/phonon.py +++ b/mp_api/client/routes/materials/phonon.py @@ -34,6 +34,7 @@ def summary_rester(self) -> SummaryRester: use_document_model=self.use_document_model, headers=self.headers, mute_progress_bars=self.mute_progress_bars, + delta_catalog=self.delta_catalog, ) return self._summary_rester @@ -175,7 +176,7 @@ def get_bandstructure_from_phonon_id( if path_type: query += f"\nAND path_convention='{path_type}'" - table = self._query_delta_single(query) + table = self._query_delta_single(query, label=ph_bs_lbl) deser = table.to_pylist(maps_as_pydicts="strict") if deser and deser[0].get("bandstructure") is not None: bs = deser[0]["bandstructure"] @@ -249,7 +250,7 @@ def get_dos_from_phonon_id( AND phonon_method='{phonon_method}' """ - table = self._query_delta_single(query) + table = self._query_delta_single(query, label=ph_dos_lbl) deser = table.to_pylist(maps_as_pydicts="strict") if deser and deser[0].get("dos") is not None: dos = deser[0]["dos"] @@ -314,7 +315,7 @@ def get_forceconstants_from_phonon_id( AND phonon_method='{phonon_method}' """ - table = self._query_delta_single(query) + table = self._query_delta_single(query, label=ph_fc_lbl) deser = table.to_pylist(maps_as_pydicts="strict") if deser and deser[0].get("force_constants") is not None: return deser[0]["force_constants"] diff --git a/mp_api/client/routes/materials/tasks.py b/mp_api/client/routes/materials/tasks.py index 1496adbd..8343b767 100644 --- a/mp_api/client/routes/materials/tasks.py +++ b/mp_api/client/routes/materials/tasks.py @@ -54,7 +54,9 @@ def get_trajectory( {predicate}; """ - traj_data = self._query_delta_single(query).to_pylist(maps_as_pydicts="strict") + traj_data = self._query_delta_single(query, label=traj_lbl).to_pylist( + maps_as_pydicts="strict" + ) if not traj_data: raise MPRestError(f"No trajectory data for {task_id} found") diff --git a/mp_api/client/routes/materials/thermo.py b/mp_api/client/routes/materials/thermo.py index 64de38cc..ec7c9069 100644 --- a/mp_api/client/routes/materials/thermo.py +++ b/mp_api/client/routes/materials/thermo.py @@ -214,7 +214,7 @@ def get_phase_diagram_from_chemsys( AND version='{version}' AND thermo_type='{validated_thermo_type}' """ - table = self._query_delta_single(query) + table = self._query_delta_single(query, label=pd_lbl) as_py = table["phase_diagram"].to_pylist(maps_as_pydicts="strict") pd: PhaseDiagram | None = None diff --git a/pyproject.toml b/pyproject.toml index 464081fe..5bbf0739 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,16 +1,31 @@ [build-system] -requires = ["setuptools", "setuptools-scm"] -build-backend = "setuptools.build_meta" +requires = ["hatchling", "uv-dynamic-versioning"] +build-backend = "hatchling.build" + +[tool.hatch.version] +source = "uv-dynamic-versioning" + +[tool.hatch.build.targets.wheel] +only-include = ["mp_api"] +dev-mode-dirs = ["."] + +[tool.hatch.build.targets.sdist] +include = [ + "mp_api", + "tests", + "pyproject.toml", +] [project] name = "mp-api" +dynamic = ["version"] +requires-python = ">=3.12" +description = "API Client for the Materials Project" +license = "BSD-3-Clause-LBNL" authors = [ { name = "The Materials Project", email = "feedback@materialsproject.org" }, ] -description = "API Client for the Materials Project" -readme = "README.md" -requires-python = ">=3.11" -license = "BSD-3-Clause-LBNL" + classifiers = [ "Programming Language :: Python :: 3", "Development Status :: 4 - Beta", @@ -19,18 +34,18 @@ classifiers = [ "Operating System :: OS Independent", "Topic :: Scientific/Engineering", ] + dependencies = [ "pymatgen>2024.2.20", "typing-extensions>=3.7.4.1", "requests>=2.23.0", "monty>=2024.12.10", - "emmet-core>=0.87.1", + "emmet-core>=0.87.3rc2,<0.87.4", "boto3", "orjson >= 3.10,<4", "pyarrow >= 20.0.0", "deltalake>=1.4.0,<1.6.0", ] -dynamic = ["version"] [project.optional-dependencies] mcp = ["fastmcp"] @@ -50,7 +65,7 @@ contribs = [ ] all = [ "custodian", - "emmet-core[all]>=0.87.1", + "emmet-core[all]>=0.87.3rc2,<0.87.4", "fastmcp", "flask", ] @@ -75,12 +90,6 @@ docs = ["sphinx"] [project.scripts] mpmcp = "mp_api.mcp.server:_run_mp_mcp_server" -[tool.setuptools.packages.find] -include = ["mp_api*"] -namespaces = true - -[tool.setuptools_scm] - [tool.pytest.ini_options] minversion = "6.0" addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"] diff --git a/requirements/requirements-ubuntu-latest_py3.11.txt b/requirements/requirements-ubuntu-latest_py3.11.txt deleted file mode 100644 index 0cb98974..00000000 --- a/requirements/requirements-ubuntu-latest_py3.11.txt +++ /dev/null @@ -1,179 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --output-file=requirements/requirements-ubuntu-latest_py3.11.txt pyproject.toml -# -annotated-types==0.8.0 - # via pydantic -arro3-core==0.8.1 - # via deltalake -bibtexparser==1.4.4 - # via pymatgen-core -blake3==1.0.9 - # via emmet-core -boto3==1.43.73 - # via mp-api (pyproject.toml) -botocore==1.43.73 - # via - # boto3 - # s3transfer -certifi==2026.7.22 - # via requests -charset-normalizer==3.5.1 - # via requests -contourpy==1.3.3 - # via matplotlib -cycler==0.12.1 - # via matplotlib -deltalake==1.5.1 - # via mp-api (pyproject.toml) -deprecated==1.3.1 - # via deltalake -emmet-core==0.87.2 - # via mp-api (pyproject.toml) -fonttools==4.63.0 - # via matplotlib -idna==3.18 - # via requests -inflect==7.5.0 - # via emmet-core -jmespath==1.1.0 - # via - # boto3 - # botocore -joblib==1.5.3 - # via pymatgen-core -kiwisolver==1.5.0 - # via matplotlib -latexcodec==3.0.1 - # via pybtex -lxml==6.1.1 - # via pymatgen-core -matplotlib==3.11.1 - # via pymatgen-core -monty==2026.7.16 - # via - # emmet-core - # mp-api (pyproject.toml) - # pymatgen-core -more-itertools==11.1.0 - # via inflect -mpmath==1.3.0 - # via sympy -narwhals==2.24.0 - # via plotly -networkx==3.6.1 - # via pymatgen-core -numpy==2.4.6 - # via - # contourpy - # matplotlib - # monty - # pandas - # pymatgen-core - # pymatgen-io-validation - # scipy - # spglib -orjson==3.12.0 - # via - # mp-api (pyproject.toml) - # pymatgen-core -packaging==26.3 - # via - # matplotlib - # plotly -palettable==3.3.3 - # via pymatgen-core -pandas==3.0.5 - # via pymatgen-core -pillow==12.3.0 - # via matplotlib -plotly==6.9.0 - # via pymatgen-core -pubchempy==1.0.5 - # via emmet-core -pyarrow==25.0.1 - # via mp-api (pyproject.toml) -pybtex==0.26.1 - # via emmet-core -pydantic==2.13.4 - # via - # emmet-core - # pydantic-settings - # pymatgen-io-validation -pydantic-core==2.46.4 - # via pydantic -pydantic-settings==2.15.0 - # via - # emmet-core - # pymatgen-io-validation -pymatgen==2026.5.4 - # via - # emmet-core - # mp-api (pyproject.toml) -pymatgen-core==2026.8.13 - # via - # pymatgen - # pymatgen-io-validation -pymatgen-io-validation==0.1.4 - # via emmet-core -pyparsing==3.3.2 - # via - # bibtexparser - # matplotlib -python-dateutil==2.9.0.post0 - # via - # botocore - # matplotlib - # pandas -python-dotenv==1.2.3 - # via pydantic-settings -pyyaml==6.0.3 - # via pybtex -requests==2.34.2 - # via - # mp-api (pyproject.toml) - # pymatgen-core - # pymatgen-io-validation -ruamel-yaml==0.19.1 - # via monty -s3transfer==0.19.2 - # via boto3 -scipy==1.17.1 - # via pymatgen-core -six==1.17.0 - # via python-dateutil -spglib==2.7.0 - # via pymatgen-core -sympy==1.14.0 - # via pymatgen-core -tabulate==0.10.0 - # via pymatgen-core -tqdm==4.70.0 - # via pymatgen-core -typeguard==4.6.0 - # via inflect -typing-extensions==4.16.0 - # via - # arro3-core - # blake3 - # emmet-core - # mp-api (pyproject.toml) - # pydantic - # pydantic-core - # spglib - # typeguard - # typing-inspection -typing-inspection==0.4.4 - # via - # pydantic - # pydantic-settings -uncertainties==3.2.3 - # via pymatgen-core -urllib3==2.8.0 - # via - # botocore - # requests -wrapt==2.3.0 - # via deprecated diff --git a/requirements/requirements-ubuntu-latest_py3.11_extras.txt b/requirements/requirements-ubuntu-latest_py3.11_extras.txt deleted file mode 100644 index 0a6920c3..00000000 --- a/requirements/requirements-ubuntu-latest_py3.11_extras.txt +++ /dev/null @@ -1,711 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --all-extras --output-file=requirements/requirements-ubuntu-latest_py3.11_extras.txt pyproject.toml -# -aiofile==3.12.3 - # via py-key-value-aio -alabaster==1.0.0 - # via sphinx -annotated-types==0.8.0 - # via pydantic -anyio==4.14.2 - # via - # httpx - # mcp - # py-key-value-aio - # sse-starlette - # starlette - # watchfiles -arro3-core==0.8.1 - # via deltalake -arrow==1.4.0 - # via isoduration -ase==3.29.0 - # via pymatgen-analysis-diffusion -ast-serialize==0.8.0 - # via mypy -asttokens==3.0.2 - # via stack-data -attrs==26.1.0 - # via - # cyclopts - # jsonschema - # jsonschema-path - # referencing -authlib==1.7.2 - # via fastmcp-slim -babel==2.18.0 - # via sphinx -backports-tarfile==1.2.0 - # via jaraco-context -beartype==0.22.9 - # via py-key-value-aio -bibtexparser==1.4.4 - # via pymatgen-core -blake3==1.0.9 - # via emmet-core -blinker==1.9.0 - # via flask -boltons==26.1.0 - # via mp-api (pyproject.toml) -boto3==1.43.73 - # via mp-api (pyproject.toml) -botocore==1.43.73 - # via - # boto3 - # s3transfer -bravado==12.0.1 - # via mp-api (pyproject.toml) -bravado-core==6.4.1 - # via bravado -cachetools==7.1.7 - # via - # mp-api (pyproject.toml) - # py-key-value-aio -caio==0.12.2 - # via aiofile -certifi==2026.7.22 - # via - # httpcore - # httpx - # requests -cffi==2.1.1 - # via cryptography -cfgv==3.5.0 - # via pre-commit -charset-normalizer==3.5.1 - # via requests -click==8.4.2 - # via - # flask - # uvicorn -contourpy==1.3.3 - # via matplotlib -coverage[toml]==7.15.4 - # via pytest-cov -cryptography==50.0.0 - # via - # authlib - # joserfc - # pyjwt - # secretstorage -custodian==2025.12.14 - # via mp-api (pyproject.toml) -cycler==0.12.1 - # via matplotlib -cyclopts==4.23.0 - # via fastmcp-slim -deltalake==1.5.1 - # via mp-api (pyproject.toml) -deprecated==1.3.1 - # via deltalake -distlib==0.4.3 - # via virtualenv -dnspython==2.8.0 - # via - # email-validator - # pyisemail - # pymongo -docstring-parser==0.18.0 - # via cyclopts -docutils==0.22.4 - # via sphinx -email-validator==2.3.0 - # via pydantic -emmet-core[all]==0.87.2 - # via mp-api (pyproject.toml) -exceptiongroup==1.3.1 - # via fastmcp-slim -execnet==2.1.2 - # via pytest-xdist -executing==2.2.1 - # via stack-data -fastmcp==3.4.7 - # via mp-api (pyproject.toml) -fastmcp-slim[client,server]==3.4.7 - # via fastmcp -filelock==3.32.3 - # via - # python-discovery - # virtualenv -filetype==1.2.0 - # via mp-api (pyproject.toml) -flake8==7.3.0 - # via mp-api (pyproject.toml) -flask==3.1.3 - # via mp-api (pyproject.toml) -flexcache==0.3 - # via pint -flexparser==0.4 - # via pint -fonttools==4.63.0 - # via matplotlib -fqdn==1.5.1 - # via jsonschema -griffelib==2.2.0 - # via fastmcp-slim -h11==0.16.0 - # via - # httpcore - # uvicorn -h5py==3.16.0 - # via phonopy -httpcore==1.0.9 - # via httpx -httpx==0.28.1 - # via - # fastmcp-slim - # mcp -httpx-sse==0.4.3 - # via mcp -identify==2.6.19 - # via pre-commit -idna==3.18 - # via - # anyio - # email-validator - # httpx - # jsonschema - # requests -imageio==2.37.4 - # via scikit-image -imagesize==2.0.0 - # via sphinx -importlib-metadata==9.0.0 - # via keyring -importlib-resources==7.1.0 - # via swagger-spec-validator -inflect==7.5.0 - # via emmet-core -iniconfig==2.3.0 - # via pytest -ipython==9.16.1 - # via mp-api (pyproject.toml) -ipython-pygments-lexers==1.1.1 - # via ipython -isoduration==20.11.0 - # via jsonschema -itsdangerous==2.2.0 - # via flask -jaraco-classes==3.4.0 - # via keyring -jaraco-context==6.1.2 - # via keyring -jaraco-functools==4.6.0 - # via keyring -jedi==0.20.0 - # via ipython -jeepney==0.9.0 - # via - # keyring - # secretstorage -jinja2==3.1.6 - # via - # flask - # sphinx -jmespath==1.1.0 - # via - # boto3 - # botocore -joblib==1.5.3 - # via - # pymatgen-analysis-diffusion - # pymatgen-core -joserfc==1.7.4 - # via - # authlib - # fastmcp-slim -json2html==1.3.0 - # via mp-api (pyproject.toml) -jsonpointer==3.1.1 - # via jsonschema -jsonref==1.1.0 - # via - # bravado-core - # fastmcp-slim -jsonschema[format-nongpl]==4.26.0 - # via - # bravado-core - # mcp - # swagger-spec-validator -jsonschema-path==0.5.0 - # via fastmcp-slim -jsonschema-specifications==2025.9.1 - # via jsonschema -keyring==25.7.0 - # via py-key-value-aio -kiwisolver==1.5.0 - # via matplotlib -lark==1.3.1 - # via rfc3987-syntax -latexcodec==3.0.1 - # via pybtex -lazy-loader==0.5 - # via scikit-image -librt==0.15.0 - # via mypy -lobsterpy==0.6.1 - # via emmet-core -lxml==6.1.1 - # via pymatgen-core -markdown-it-py==4.2.0 - # via rich -markupsafe==3.0.3 - # via - # flask - # jinja2 - # werkzeug -matplotlib==3.11.1 - # via - # ase - # phonopy - # pymatgen-core - # seaborn -matplotlib-inline==0.2.2 - # via ipython -mccabe==0.7.0 - # via flake8 -mcp==1.29.0 - # via fastmcp-slim -mdurl==0.1.2 - # via markdown-it-py -monotonic==1.6 - # via bravado -monty==2026.7.16 - # via - # custodian - # emmet-core - # mp-api (pyproject.toml) - # pymatgen-core -more-itertools==11.1.0 - # via - # inflect - # jaraco-classes - # jaraco-functools -mp-pyrho==0.5.1 - # via pymatgen-analysis-defects -mpmath==1.3.0 - # via sympy -msgpack==1.2.1 - # via - # bravado - # bravado-core -mypy==2.3.1 - # via mp-api (pyproject.toml) -mypy-extensions==1.1.0 - # via - # mp-api (pyproject.toml) - # mypy -narwhals==2.24.0 - # via plotly -networkx==3.6.1 - # via - # pymatgen-core - # scikit-image -nodeenv==1.10.0 - # via pre-commit -numpy==2.4.6 - # via - # ase - # contourpy - # h5py - # imageio - # lobsterpy - # matplotlib - # monty - # pandas - # phonopy - # pymatgen-analysis-defects - # pymatgen-analysis-diffusion - # pymatgen-core - # pymatgen-io-validation - # scikit-image - # scipy - # seaborn - # seekpath - # shapely - # spglib - # symfc - # tifffile -openapi-pydantic==0.5.1 - # via fastmcp-slim -opentelemetry-api==1.44.0 - # via fastmcp-slim -orjson==3.12.0 - # via - # mp-api (pyproject.toml) - # pymatgen-core -packaging==26.3 - # via - # fastmcp-slim - # lazy-loader - # matplotlib - # plotly - # pytest - # scikit-image - # sphinx -palettable==3.3.3 - # via pymatgen-core -pandas==3.0.5 - # via - # pymatgen-core - # seaborn -parso==0.8.7 - # via jedi -pathable==0.6.0 - # via jsonschema-path -pathspec==1.1.1 - # via mypy -pexpect==4.9.0 - # via ipython -phonopy==4.4.0 - # via emmet-core -phonors==0.3.0 - # via phonopy -pillow==12.3.0 - # via - # imageio - # matplotlib - # scikit-image -pint==0.25.3 - # via mp-api (pyproject.toml) -platformdirs==4.11.3 - # via - # fastmcp-slim - # pint - # virtualenv -plotly==6.9.0 - # via pymatgen-core -pluggy==1.6.0 - # via - # pytest - # pytest-cov -pre-commit==4.6.2 - # via mp-api (pyproject.toml) -prompt-toolkit==3.0.53 - # via ipython -psutil==7.2.2 - # via - # custodian - # ipython -ptyprocess==0.7.0 - # via pexpect -pubchempy==1.0.5 - # via emmet-core -pure-eval==0.2.3 - # via stack-data -py-key-value-aio[filetree,keyring,memory]==0.4.5 - # via fastmcp-slim -pyarrow==25.0.1 - # via - # emmet-core - # mp-api (pyproject.toml) -pybtex==0.26.1 - # via emmet-core -pycodcif==3.0.1 - # via emmet-core -pycodestyle==2.14.0 - # via - # flake8 - # mp-api (pyproject.toml) -pycparser==3.0 - # via cffi -pydantic[email]==2.13.4 - # via - # emmet-core - # fastmcp-slim - # mcp - # openapi-pydantic - # pydantic-settings - # pymatgen-io-validation -pydantic-core==2.46.4 - # via pydantic -pydantic-settings==2.15.0 - # via - # emmet-core - # fastmcp-slim - # mcp - # pymatgen-io-validation -pyflakes==3.4.0 - # via flake8 -pygments==2.21.0 - # via - # ipython - # ipython-pygments-lexers - # pytest - # rich - # rich-rst - # sphinx -pyisemail==2.0.1 - # via mp-api (pyproject.toml) -pyjwt[crypto]==2.15.1 - # via mcp -pymatgen==2026.5.4 - # via - # emmet-core - # lobsterpy - # mp-api (pyproject.toml) - # mp-pyrho - # pymatgen-analysis-alloys - # pymatgen-analysis-defects - # pymatgen-analysis-diffusion -pymatgen-analysis-alloys==0.0.9 - # via emmet-core -pymatgen-analysis-defects==2026.3.20 - # via emmet-core -pymatgen-analysis-diffusion==2025.11.15 - # via emmet-core -pymatgen-core==2026.7.16 - # via - # emmet-core - # pymatgen - # pymatgen-io-validation -pymatgen-io-validation==0.1.4 - # via emmet-core -pymongo==4.17.0 - # via - # emmet-core - # mp-api (pyproject.toml) -pyparsing==3.3.2 - # via - # bibtexparser - # matplotlib -pyperclip==1.11.0 - # via fastmcp-slim -pytest==9.1.1 - # via - # mp-api (pyproject.toml) - # pytest-asyncio - # pytest-cov - # pytest-mock - # pytest-xdist -pytest-asyncio==1.4.0 - # via mp-api (pyproject.toml) -pytest-cov==7.1.0 - # via mp-api (pyproject.toml) -pytest-mock==3.15.1 - # via mp-api (pyproject.toml) -pytest-xdist==3.8.0 - # via mp-api (pyproject.toml) -python-dateutil==2.9.0.post0 - # via - # arrow - # botocore - # bravado - # bravado-core - # matplotlib - # pandas -python-discovery==1.5.2 - # via virtualenv -python-dotenv==1.2.3 - # via - # fastmcp-slim - # pydantic-settings -python-multipart==0.0.32 - # via - # fastmcp-slim - # mcp -pytz==2026.3.post1 - # via bravado-core -pyyaml==6.0.3 - # via - # bravado - # bravado-core - # fastmcp-slim - # jsonschema-path - # phonopy - # pre-commit - # pybtex - # swagger-spec-validator -referencing==0.37.0 - # via - # jsonschema - # jsonschema-path - # jsonschema-specifications -requests==2.34.2 - # via - # bravado - # bravado-core - # mp-api (pyproject.toml) - # pymatgen-core - # pymatgen-io-validation - # requests-futures - # sphinx -requests-futures==1.0.2 - # via mp-api (pyproject.toml) -rfc3339-validator==0.1.4 - # via jsonschema -rfc3986-validator==0.1.1 - # via jsonschema -rfc3987-syntax==1.1.0 - # via jsonschema -rich==15.0.0 - # via - # cyclopts - # fastmcp-slim - # rich-rst -rich-rst==2.1.0 - # via cyclopts -roman-numerals==4.1.0 - # via sphinx -rpds-py==2026.6.3 - # via - # jsonschema - # referencing -ruamel-yaml==0.19.1 - # via - # custodian - # monty -s3transfer==0.19.2 - # via boto3 -scikit-image==0.26.0 - # via pymatgen-analysis-defects -scipy==1.17.1 - # via - # ase - # pymatgen-core - # scikit-image - # symfc -seaborn==0.13.2 - # via pymatgen-analysis-diffusion -secretstorage==3.5.0 - # via keyring -seekpath==2.2.1 - # via emmet-core -shapely==2.1.2 - # via pymatgen-analysis-alloys -simplejson==4.1.1 - # via - # bravado - # bravado-core -six==1.17.0 - # via - # bravado - # bravado-core - # python-dateutil - # rfc3339-validator -snowballstemmer==3.1.1 - # via sphinx -spglib==2.7.0 - # via - # phonopy - # pymatgen-core - # seekpath - # symfc -sphinx==9.0.4 - # via mp-api (pyproject.toml) -sphinxcontrib-applehelp==2.0.0 - # via sphinx -sphinxcontrib-devhelp==2.0.0 - # via sphinx -sphinxcontrib-htmlhelp==2.1.0 - # via sphinx -sphinxcontrib-jsmath==1.0.1 - # via sphinx -sphinxcontrib-qthelp==2.0.0 - # via sphinx -sphinxcontrib-serializinghtml==2.0.0 - # via sphinx -sse-starlette==3.4.8 - # via mcp -stack-data==0.6.3 - # via ipython -starlette==1.6.0 - # via - # fastmcp-slim - # mcp - # sse-starlette -swagger-spec-validator==3.0.4 - # via - # bravado-core - # mp-api (pyproject.toml) -symfc==1.7.3 - # via phonopy -sympy==1.14.0 - # via pymatgen-core -tabulate==0.10.0 - # via pymatgen-core -tifffile==2026.3.3 - # via scikit-image -tqdm==4.70.0 - # via pymatgen-core -traitlets==5.16.1 - # via - # ipython - # matplotlib-inline -typeguard==4.6.0 - # via inflect -types-requests==2.33.0.20260712 - # via mp-api (pyproject.toml) -types-setuptools==84.0.0.20260812 - # via mp-api (pyproject.toml) -types-tqdm==4.70.0.20260805 - # via mp-api (pyproject.toml) -typing-extensions==4.16.0 - # via - # anyio - # arro3-core - # ase - # blake3 - # bravado - # emmet-core - # exceptiongroup - # fastmcp-slim - # flexcache - # flexparser - # ipython - # mcp - # mp-api (pyproject.toml) - # mypy - # opentelemetry-api - # pint - # py-key-value-aio - # pydantic - # pydantic-core - # pytest-asyncio - # referencing - # spglib - # starlette - # swagger-spec-validator - # typeguard - # typing-inspection -typing-inspection==0.4.4 - # via - # mcp - # pydantic - # pydantic-settings -tzdata==2026.3 - # via arrow -uncalled-for==0.4.0 - # via fastmcp-slim -uncertainties==3.2.3 - # via pymatgen-core -uri-template==1.3.0 - # via jsonschema -urllib3==2.8.0 - # via - # botocore - # requests - # types-requests -uvicorn==0.52.3 - # via - # fastmcp-slim - # mcp -virtualenv==21.7.13 - # via pre-commit -watchfiles==1.2.0 - # via fastmcp-slim -wcwidth==0.8.2 - # via prompt-toolkit -webcolors==25.10.0 - # via jsonschema -websockets==17.0.1 - # via fastmcp-slim -werkzeug==3.1.8 - # via flask -wrapt==2.3.0 - # via deprecated -zipp==4.1.0 - # via importlib-metadata diff --git a/requirements/requirements-ubuntu-latest_py3.12.txt b/requirements/requirements-ubuntu-latest_py3.12.txt index 7dfa7c17..e353464f 100644 --- a/requirements/requirements-ubuntu-latest_py3.12.txt +++ b/requirements/requirements-ubuntu-latest_py3.12.txt @@ -2,39 +2,41 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile --output-file=requirements/requirements-ubuntu-latest_py3.12.txt pyproject.toml +# pip-compile --no-index --output-file=requirements/requirements-ubuntu-latest_py3.12.txt pyproject.toml # annotated-types==0.8.0 # via pydantic -arro3-core==0.8.1 +arro3-core==0.9.0 # via deltalake bibtexparser==1.4.4 # via pymatgen-core -blake3==1.0.9 +blake3==1.0.10 # via emmet-core -boto3==1.43.73 +boto3==1.43.108 # via mp-api (pyproject.toml) -botocore==1.43.73 +botocore==1.43.108 # via # boto3 # s3transfer certifi==2026.7.22 # via requests -charset-normalizer==3.5.1 +charset-normalizer==3.5.2 # via requests -contourpy==1.3.3 +cloudpickle==3.1.2 + # via joblib +contourpy==1.4.0 # via matplotlib cycler==0.12.1 # via matplotlib deltalake==1.5.1 # via mp-api (pyproject.toml) -deprecated==1.3.1 +deprecated==3.0.0 # via deltalake -emmet-core==0.87.2 +emmet-core==0.87.3rc2 # via mp-api (pyproject.toml) -fonttools==4.63.0 +fonttools==4.66.1 # via matplotlib -idna==3.18 +idna==3.20 # via requests inflect==7.5.0 # via emmet-core @@ -42,15 +44,15 @@ jmespath==1.1.0 # via # boto3 # botocore -joblib==1.5.3 +joblib==1.6.0 # via pymatgen-core -kiwisolver==1.5.0 +kiwisolver==1.5.1 # via matplotlib latexcodec==3.0.1 # via pybtex -lxml==6.1.1 +lxml==6.1.3 # via pymatgen-core -matplotlib==3.11.1 +matplotlib==3.11.2 # via pymatgen-core monty==2026.7.16 # via @@ -61,11 +63,11 @@ more-itertools==11.1.0 # via inflect mpmath==1.3.0 # via sympy -narwhals==2.24.0 +narwhals==2.26.0 # via plotly -networkx==3.6.1 +networkx==3.7 # via pymatgen-core -numpy==2.5.2 +numpy==2.5.3 # via # contourpy # matplotlib @@ -85,11 +87,11 @@ packaging==26.3 # plotly palettable==3.3.3 # via pymatgen-core -pandas==3.0.5 +pandas==3.0.6 # via pymatgen-core pillow==12.3.0 # via matplotlib -plotly==6.9.0 +plotly==7.1.0 # via pymatgen-core pubchempy==1.0.5 # via emmet-core @@ -97,28 +99,29 @@ pyarrow==25.0.1 # via mp-api (pyproject.toml) pybtex==0.26.1 # via emmet-core -pydantic==2.13.4 +pydantic==2.13.5 # via # emmet-core # pydantic-settings # pymatgen-io-validation -pydantic-core==2.46.4 +pydantic-core==2.46.5 # via pydantic pydantic-settings==2.15.0 # via # emmet-core # pymatgen-io-validation -pymatgen==2026.5.4 +pymatgen==2026.9.24 # via # emmet-core # mp-api (pyproject.toml) -pymatgen-core==2026.8.13 +pymatgen-core==2026.10.2 # via + # emmet-core # pymatgen # pymatgen-io-validation pymatgen-io-validation==0.1.4 # via emmet-core -pyparsing==3.3.2 +pyparsing==3.3.3 # via # bibtexparser # matplotlib @@ -127,7 +130,7 @@ python-dateutil==2.9.0.post0 # botocore # matplotlib # pandas -python-dotenv==1.2.3 +python-dotenv==1.2.4 # via pydantic-settings pyyaml==6.0.3 # via pybtex @@ -140,7 +143,7 @@ ruamel-yaml==0.19.1 # via monty s3transfer==0.19.2 # via boto3 -scipy==1.18.0 +scipy==1.18.1 # via pymatgen-core six==1.17.0 # via python-dateutil @@ -150,7 +153,7 @@ sympy==1.14.0 # via pymatgen-core tabulate==0.10.0 # via pymatgen-core -tqdm==4.70.0 +tqdm==4.70.1 # via pymatgen-core typeguard==4.6.0 # via inflect @@ -173,5 +176,5 @@ urllib3==2.8.0 # via # botocore # requests -wrapt==2.3.0 +wrapt==2.5.0 # via deprecated diff --git a/requirements/requirements-ubuntu-latest_py3.12_extras.txt b/requirements/requirements-ubuntu-latest_py3.12_extras.txt index ee4c38ff..711b9139 100644 --- a/requirements/requirements-ubuntu-latest_py3.12_extras.txt +++ b/requirements/requirements-ubuntu-latest_py3.12_extras.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile --all-extras --output-file=requirements/requirements-ubuntu-latest_py3.12_extras.txt pyproject.toml +# pip-compile --all-extras --no-index --output-file=requirements/requirements-ubuntu-latest_py3.12_extras.txt pyproject.toml # aiofile==3.12.3 # via py-key-value-aio @@ -10,21 +10,21 @@ alabaster==1.0.0 # via sphinx annotated-types==0.8.0 # via pydantic -anyio==4.14.2 +anyio==4.15.1 # via - # httpx + # httpx2 # mcp # py-key-value-aio # sse-starlette # starlette # watchfiles -arro3-core==0.8.1 +arro3-core==0.9.0 # via deltalake arrow==1.4.0 # via isoduration ase==3.29.0 # via pymatgen-analysis-diffusion -ast-serialize==0.8.0 +ast-serialize==0.12.0 # via mypy asttokens==3.0.2 # via stack-data @@ -34,7 +34,7 @@ attrs==26.1.0 # jsonschema # jsonschema-path # referencing -authlib==1.7.2 +authlib==1.8.0 # via fastmcp-slim babel==2.18.0 # via sphinx @@ -42,15 +42,15 @@ beartype==0.22.9 # via py-key-value-aio bibtexparser==1.4.4 # via pymatgen-core -blake3==1.0.9 +blake3==1.0.10 # via emmet-core blinker==1.9.0 # via flask -boltons==26.1.0 +boltons==26.2.0 # via mp-api (pyproject.toml) -boto3==1.43.73 +boto3==1.43.108 # via mp-api (pyproject.toml) -botocore==1.43.73 +botocore==1.43.108 # via # boto3 # s3transfer @@ -58,32 +58,31 @@ bravado==12.0.1 # via mp-api (pyproject.toml) bravado-core==6.4.1 # via bravado -cachetools==7.1.7 +cachetools==7.2.0 # via # mp-api (pyproject.toml) # py-key-value-aio -caio==0.12.2 +caio==0.12.9 # via aiofile certifi==2026.7.22 - # via - # httpcore - # httpx - # requests + # via requests cffi==2.1.1 # via cryptography cfgv==3.5.0 # via pre-commit -charset-normalizer==3.5.1 +charset-normalizer==3.5.2 # via requests -click==8.4.2 +click==8.5.0 # via # flask # uvicorn -contourpy==1.3.3 +cloudpickle==3.1.2 + # via joblib +contourpy==1.4.0 # via matplotlib -coverage[toml]==7.15.4 +coverage[toml]==7.16.2 # via pytest-cov -cryptography==50.0.0 +cryptography==50.0.2 # via # authlib # joserfc @@ -93,11 +92,11 @@ custodian==2025.12.14 # via mp-api (pyproject.toml) cycler==0.12.1 # via matplotlib -cyclopts==4.23.0 +cyclopts==5.1.1 # via fastmcp-slim deltalake==1.5.1 # via mp-api (pyproject.toml) -deprecated==1.3.1 +deprecated==3.0.0 # via deltalake distlib==0.4.3 # via virtualenv @@ -112,7 +111,7 @@ docutils==0.22.4 # via sphinx email-validator==2.3.0 # via pydantic -emmet-core[all]==0.87.2 +emmet-core[all]==0.87.3rc2 # via mp-api (pyproject.toml) exceptiongroup==1.3.1 # via fastmcp-slim @@ -120,17 +119,17 @@ execnet==2.1.2 # via pytest-xdist executing==2.2.1 # via stack-data -fastmcp==3.4.7 +fastmcp==4.0.10 # via mp-api (pyproject.toml) -fastmcp-slim[client,server]==3.4.7 +fastmcp-slim[client,server]==4.0.10 # via fastmcp -filelock==3.32.3 +filelock==4.0.9 # via # python-discovery # virtualenv filetype==1.2.0 # via mp-api (pyproject.toml) -flake8==7.3.0 +flake8==7.4.1 # via mp-api (pyproject.toml) flask==3.1.3 # via mp-api (pyproject.toml) @@ -138,38 +137,36 @@ flexcache==0.3 # via pint flexparser==0.4 # via pint -fonttools==4.63.0 +fonttools==4.66.1 # via matplotlib -fqdn==1.5.1 +fqdn==1.6.0 # via jsonschema -griffelib==2.2.0 +griffelib==2.3.0 # via fastmcp-slim h11==0.16.0 # via - # httpcore + # httpcore2 # uvicorn h5py==3.16.0 # via phonopy -httpcore==1.0.9 - # via httpx -httpx==0.28.1 +httpcore2==2.13.1 + # via httpx2 +httpx2==2.13.1 # via # fastmcp-slim # mcp -httpx-sse==0.4.3 - # via mcp -identify==2.6.19 +identify==2.6.20 # via pre-commit -idna==3.18 +idna==3.20 # via # anyio # email-validator - # httpx + # httpx2 # jsonschema # requests -imageio==2.37.4 +imageio==2.38.0 # via scikit-image -imagesize==2.0.0 +imagesize==2.0.1 # via sphinx importlib-resources==7.1.0 # via swagger-spec-validator @@ -177,7 +174,7 @@ inflect==7.5.0 # via emmet-core iniconfig==2.3.0 # via pytest -ipython==9.16.1 +ipython==9.17.1 # via mp-api (pyproject.toml) ipython-pygments-lexers==1.1.1 # via ipython @@ -205,11 +202,11 @@ jmespath==1.1.0 # via # boto3 # botocore -joblib==1.5.3 +joblib==1.6.0 # via # pymatgen-analysis-diffusion # pymatgen-core -joserfc==1.7.4 +joserfc==1.7.5 # via # authlib # fastmcp-slim @@ -232,19 +229,19 @@ jsonschema-specifications==2025.9.1 # via jsonschema keyring==25.7.0 # via py-key-value-aio -kiwisolver==1.5.0 +kiwisolver==1.5.1 # via matplotlib lark==1.3.1 # via rfc3987-syntax latexcodec==3.0.1 # via pybtex -lazy-loader==0.5 +lazy-loader==0.6 # via scikit-image -librt==0.15.0 +librt==0.16.0 # via mypy lobsterpy==0.6.1 # via emmet-core -lxml==6.1.1 +lxml==6.1.3 # via pymatgen-core markdown-it-py==4.2.0 # via rich @@ -253,7 +250,7 @@ markupsafe==3.0.3 # flask # jinja2 # werkzeug -matplotlib==3.11.1 +matplotlib==3.11.2 # via # ase # phonopy @@ -263,8 +260,12 @@ matplotlib-inline==0.2.2 # via ipython mccabe==0.7.0 # via flake8 -mcp==1.29.0 +mcp==2.2.0 # via fastmcp-slim +mcp-types==2.2.0 + # via + # fastmcp-slim + # mcp mdurl==0.1.2 # via markdown-it-py monotonic==1.6 @@ -284,25 +285,25 @@ mp-pyrho==0.5.1 # via pymatgen-analysis-defects mpmath==1.3.0 # via sympy -msgpack==1.2.1 +msgpack==1.2.3 # via # bravado # bravado-core -mypy==2.3.1 +mypy==2.4.0 # via mp-api (pyproject.toml) mypy-extensions==1.1.0 # via # mp-api (pyproject.toml) # mypy -narwhals==2.24.0 +narwhals==2.26.0 # via plotly -networkx==3.6.1 +networkx==3.7 # via # pymatgen-core # scikit-image -nodeenv==1.10.0 +nodeenv==1.11.0 # via pre-commit -numpy==2.5.2 +numpy==2.5.3 # via # ase # contourpy @@ -325,10 +326,12 @@ numpy==2.5.2 # spglib # symfc # tifffile -openapi-pydantic==0.5.1 - # via fastmcp-slim -opentelemetry-api==1.44.0 +openapi-pydantic==0.6.0 # via fastmcp-slim +opentelemetry-api==1.45.0 + # via + # fastmcp-slim + # mcp orjson==3.12.0 # via # mp-api (pyproject.toml) @@ -341,10 +344,12 @@ packaging==26.3 # plotly # pytest # scikit-image + # seekpath # sphinx + # virtualenv palettable==3.3.3 # via pymatgen-core -pandas==3.0.5 +pandas==3.0.6 # via # pymatgen-core # seaborn @@ -356,23 +361,23 @@ pathspec==1.1.1 # via mypy pexpect==4.9.0 # via ipython -phonopy==4.4.0 +phonopy==4.8.0 # via emmet-core -phonors==0.3.0 +phonors==0.6.0 # via phonopy pillow==12.3.0 # via # imageio # matplotlib # scikit-image -pint==0.25.3 +pint==0.26.1 # via mp-api (pyproject.toml) -platformdirs==4.11.3 +platformdirs==4.12.2 # via # fastmcp-slim # pint # virtualenv -plotly==6.9.0 +plotly==7.1.0 # via pymatgen-core pluggy==1.6.0 # via @@ -390,9 +395,9 @@ ptyprocess==0.7.0 # via pexpect pubchempy==1.0.5 # via emmet-core -pure-eval==0.2.3 +pure-eval==0.2.4 # via stack-data -py-key-value-aio[filetree,keyring,memory]==0.4.5 +py-key-value-aio[filetree,keyring,memory]==0.4.6 # via fastmcp-slim pyarrow==25.0.1 # via @@ -402,29 +407,29 @@ pybtex==0.26.1 # via emmet-core pycodcif==3.0.1 # via emmet-core -pycodestyle==2.14.0 +pycodestyle==2.15.0 # via # flake8 # mp-api (pyproject.toml) pycparser==3.0 # via cffi -pydantic[email]==2.13.4 +pydantic[email]==2.13.5 # via # emmet-core # fastmcp-slim # mcp + # mcp-types # openapi-pydantic # pydantic-settings # pymatgen-io-validation -pydantic-core==2.46.4 +pydantic-core==2.46.5 # via pydantic pydantic-settings==2.15.0 # via # emmet-core # fastmcp-slim - # mcp # pymatgen-io-validation -pyflakes==3.4.0 +pyflakes==4.0.1 # via flake8 pygments==2.21.0 # via @@ -438,7 +443,7 @@ pyisemail==2.0.1 # via mp-api (pyproject.toml) pyjwt[crypto]==2.15.1 # via mcp -pymatgen==2026.5.4 +pymatgen==2026.9.24 # via # emmet-core # lobsterpy @@ -451,20 +456,20 @@ pymatgen-analysis-alloys==0.0.9 # via emmet-core pymatgen-analysis-defects==2026.3.20 # via emmet-core -pymatgen-analysis-diffusion==2025.11.15 +pymatgen-analysis-diffusion==2026.9.23 # via emmet-core -pymatgen-core==2026.7.16 +pymatgen-core==2026.10.2 # via # emmet-core # pymatgen # pymatgen-io-validation pymatgen-io-validation==0.1.4 # via emmet-core -pymongo==4.17.0 +pymongo==4.18.2 # via # emmet-core # mp-api (pyproject.toml) -pyparsing==3.3.2 +pyparsing==3.3.3 # via # bibtexparser # matplotlib @@ -481,7 +486,7 @@ pytest-asyncio==1.4.0 # via mp-api (pyproject.toml) pytest-cov==7.1.0 # via mp-api (pyproject.toml) -pytest-mock==3.15.1 +pytest-mock==3.16.0 # via mp-api (pyproject.toml) pytest-xdist==3.8.0 # via mp-api (pyproject.toml) @@ -493,9 +498,9 @@ python-dateutil==2.9.0.post0 # bravado-core # matplotlib # pandas -python-discovery==1.5.2 +python-discovery==1.6.1 # via virtualenv -python-dotenv==1.2.3 +python-dotenv==1.2.4 # via # fastmcp-slim # pydantic-settings @@ -503,7 +508,7 @@ python-multipart==0.0.32 # via # fastmcp-slim # mcp -pytz==2026.3.post1 +pytz==2026.4 # via bravado-core pyyaml==6.0.3 # via @@ -529,7 +534,7 @@ requests==2.34.2 # pymatgen-io-validation # requests-futures # sphinx -requests-futures==1.0.2 +requests-futures==1.1.0 # via mp-api (pyproject.toml) rfc3339-validator==0.1.4 # via jsonschema @@ -542,7 +547,7 @@ rich==15.0.0 # cyclopts # fastmcp-slim # rich-rst -rich-rst==2.1.0 +rich-rst==2.2.0 # via cyclopts roman-numerals==4.1.0 # via sphinx @@ -558,7 +563,7 @@ s3transfer==0.19.2 # via boto3 scikit-image==0.26.0 # via pymatgen-analysis-defects -scipy==1.18.0 +scipy==1.18.1 # via # ase # pymatgen-core @@ -568,11 +573,11 @@ seaborn==0.13.2 # via pymatgen-analysis-diffusion secretstorage==3.5.0 # via keyring -seekpath==2.2.1 +seekpath==2.2.2 # via emmet-core shapely==2.1.2 # via pymatgen-analysis-alloys -simplejson==4.1.1 +simplejson==4.1.2 # via # bravado # bravado-core @@ -604,11 +609,11 @@ sphinxcontrib-qthelp==2.0.0 # via sphinx sphinxcontrib-serializinghtml==2.0.0 # via sphinx -sse-starlette==3.4.8 +sse-starlette==3.5.0 # via mcp stack-data==0.6.3 # via ipython -starlette==1.6.0 +starlette==1.7.0 # via # fastmcp-slim # mcp @@ -623,21 +628,25 @@ sympy==1.14.0 # via pymatgen-core tabulate==0.10.0 # via pymatgen-core -tifffile==2026.8.16 +tifffile==2026.9.20 # via scikit-image -tqdm==4.70.0 +tqdm==4.70.1 # via pymatgen-core traitlets==5.16.1 # via # ipython # matplotlib-inline +truststore==0.10.4 + # via + # httpcore2 + # httpx2 typeguard==4.6.0 # via inflect -types-requests==2.33.0.20260712 +types-requests==2.33.0.20260906 # via mp-api (pyproject.toml) types-setuptools==84.0.0.20260812 # via mp-api (pyproject.toml) -types-tqdm==4.70.0.20260805 +types-tqdm==4.70.0.20260906 # via mp-api (pyproject.toml) typing-extensions==4.16.0 # via @@ -649,7 +658,9 @@ typing-extensions==4.16.0 # fastmcp-slim # flexcache # flexparser + # httpx2 # mcp + # mcp-types # mp-api (pyproject.toml) # mypy # opentelemetry-api @@ -669,7 +680,7 @@ typing-inspection==0.4.4 # mcp # pydantic # pydantic-settings -tzdata==2026.3 +tzdata==2026.4 # via arrow uncalled-for==0.4.0 # via fastmcp-slim @@ -682,21 +693,21 @@ urllib3==2.8.0 # botocore # requests # types-requests -uvicorn==0.52.3 +uvicorn==0.54.0 # via # fastmcp-slim # mcp -virtualenv==21.7.13 +virtualenv==21.14.5 # via pre-commit -watchfiles==1.2.0 +watchfiles==1.3.0 # via fastmcp-slim -wcwidth==0.8.2 +wcwidth==0.9.1 # via prompt-toolkit webcolors==25.10.0 # via jsonschema -websockets==17.0.1 +websockets==17.1 # via fastmcp-slim -werkzeug==3.1.8 +werkzeug==3.1.9 # via flask -wrapt==2.3.0 +wrapt==2.5.0 # via deprecated diff --git a/requirements/requirements-ubuntu-latest_py3.13.txt b/requirements/requirements-ubuntu-latest_py3.13.txt index 049b4955..742dd107 100644 --- a/requirements/requirements-ubuntu-latest_py3.13.txt +++ b/requirements/requirements-ubuntu-latest_py3.13.txt @@ -2,39 +2,41 @@ # This file is autogenerated by pip-compile with Python 3.13 # by the following command: # -# pip-compile --output-file=requirements/requirements-ubuntu-latest_py3.13.txt pyproject.toml +# pip-compile --no-index --output-file=requirements/requirements-ubuntu-latest_py3.13.txt pyproject.toml # annotated-types==0.8.0 # via pydantic -arro3-core==0.8.1 +arro3-core==0.9.0 # via deltalake bibtexparser==1.4.4 # via pymatgen-core -blake3==1.0.9 +blake3==1.0.10 # via emmet-core -boto3==1.43.73 +boto3==1.43.108 # via mp-api (pyproject.toml) -botocore==1.43.73 +botocore==1.43.108 # via # boto3 # s3transfer certifi==2026.7.22 # via requests -charset-normalizer==3.5.1 +charset-normalizer==3.5.2 # via requests -contourpy==1.3.3 +cloudpickle==3.1.2 + # via joblib +contourpy==1.4.0 # via matplotlib cycler==0.12.1 # via matplotlib deltalake==1.5.1 # via mp-api (pyproject.toml) -deprecated==1.3.1 +deprecated==3.0.0 # via deltalake -emmet-core==0.87.2 +emmet-core==0.87.3rc2 # via mp-api (pyproject.toml) -fonttools==4.63.0 +fonttools==4.66.1 # via matplotlib -idna==3.18 +idna==3.20 # via requests inflect==7.5.0 # via emmet-core @@ -42,15 +44,15 @@ jmespath==1.1.0 # via # boto3 # botocore -joblib==1.5.3 +joblib==1.6.0 # via pymatgen-core -kiwisolver==1.5.0 +kiwisolver==1.5.1 # via matplotlib latexcodec==3.0.1 # via pybtex -lxml==6.1.1 +lxml==6.1.3 # via pymatgen-core -matplotlib==3.11.1 +matplotlib==3.11.2 # via pymatgen-core monty==2026.7.16 # via @@ -61,11 +63,11 @@ more-itertools==11.1.0 # via inflect mpmath==1.3.0 # via sympy -narwhals==2.24.0 +narwhals==2.26.0 # via plotly -networkx==3.6.1 +networkx==3.7 # via pymatgen-core -numpy==2.5.2 +numpy==2.5.3 # via # contourpy # matplotlib @@ -85,11 +87,11 @@ packaging==26.3 # plotly palettable==3.3.3 # via pymatgen-core -pandas==3.0.5 +pandas==3.0.6 # via pymatgen-core pillow==12.3.0 # via matplotlib -plotly==6.9.0 +plotly==7.1.0 # via pymatgen-core pubchempy==1.0.5 # via emmet-core @@ -97,28 +99,29 @@ pyarrow==25.0.1 # via mp-api (pyproject.toml) pybtex==0.26.1 # via emmet-core -pydantic==2.13.4 +pydantic==2.13.5 # via # emmet-core # pydantic-settings # pymatgen-io-validation -pydantic-core==2.46.4 +pydantic-core==2.46.5 # via pydantic pydantic-settings==2.15.0 # via # emmet-core # pymatgen-io-validation -pymatgen==2026.5.4 +pymatgen==2026.9.24 # via # emmet-core # mp-api (pyproject.toml) -pymatgen-core==2026.8.13 +pymatgen-core==2026.10.2 # via + # emmet-core # pymatgen # pymatgen-io-validation pymatgen-io-validation==0.1.4 # via emmet-core -pyparsing==3.3.2 +pyparsing==3.3.3 # via # bibtexparser # matplotlib @@ -127,7 +130,7 @@ python-dateutil==2.9.0.post0 # botocore # matplotlib # pandas -python-dotenv==1.2.3 +python-dotenv==1.2.4 # via pydantic-settings pyyaml==6.0.3 # via pybtex @@ -140,7 +143,7 @@ ruamel-yaml==0.19.1 # via monty s3transfer==0.19.2 # via boto3 -scipy==1.18.0 +scipy==1.18.1 # via pymatgen-core six==1.17.0 # via python-dateutil @@ -150,7 +153,7 @@ sympy==1.14.0 # via pymatgen-core tabulate==0.10.0 # via pymatgen-core -tqdm==4.70.0 +tqdm==4.70.1 # via pymatgen-core typeguard==4.6.0 # via inflect @@ -172,5 +175,5 @@ urllib3==2.8.0 # via # botocore # requests -wrapt==2.3.0 +wrapt==2.5.0 # via deprecated diff --git a/requirements/requirements-ubuntu-latest_py3.13_extras.txt b/requirements/requirements-ubuntu-latest_py3.13_extras.txt index 811be2b6..a1c03312 100644 --- a/requirements/requirements-ubuntu-latest_py3.13_extras.txt +++ b/requirements/requirements-ubuntu-latest_py3.13_extras.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.13 # by the following command: # -# pip-compile --all-extras --output-file=requirements/requirements-ubuntu-latest_py3.13_extras.txt pyproject.toml +# pip-compile --all-extras --no-index --output-file=requirements/requirements-ubuntu-latest_py3.13_extras.txt pyproject.toml # aiofile==3.12.3 # via py-key-value-aio @@ -10,21 +10,21 @@ alabaster==1.0.0 # via sphinx annotated-types==0.8.0 # via pydantic -anyio==4.14.2 +anyio==4.15.1 # via - # httpx + # httpx2 # mcp # py-key-value-aio # sse-starlette # starlette # watchfiles -arro3-core==0.8.1 +arro3-core==0.9.0 # via deltalake arrow==1.4.0 # via isoduration ase==3.29.0 # via pymatgen-analysis-diffusion -ast-serialize==0.8.0 +ast-serialize==0.12.0 # via mypy asttokens==3.0.2 # via stack-data @@ -34,7 +34,7 @@ attrs==26.1.0 # jsonschema # jsonschema-path # referencing -authlib==1.7.2 +authlib==1.8.0 # via fastmcp-slim babel==2.18.0 # via sphinx @@ -42,15 +42,15 @@ beartype==0.22.9 # via py-key-value-aio bibtexparser==1.4.4 # via pymatgen-core -blake3==1.0.9 +blake3==1.0.10 # via emmet-core blinker==1.9.0 # via flask -boltons==26.1.0 +boltons==26.2.0 # via mp-api (pyproject.toml) -boto3==1.43.73 +boto3==1.43.108 # via mp-api (pyproject.toml) -botocore==1.43.73 +botocore==1.43.108 # via # boto3 # s3transfer @@ -58,32 +58,31 @@ bravado==12.0.1 # via mp-api (pyproject.toml) bravado-core==6.4.1 # via bravado -cachetools==7.1.7 +cachetools==7.2.0 # via # mp-api (pyproject.toml) # py-key-value-aio -caio==0.12.2 +caio==0.12.9 # via aiofile certifi==2026.7.22 - # via - # httpcore - # httpx - # requests + # via requests cffi==2.1.1 # via cryptography cfgv==3.5.0 # via pre-commit -charset-normalizer==3.5.1 +charset-normalizer==3.5.2 # via requests -click==8.4.2 +click==8.5.0 # via # flask # uvicorn -contourpy==1.3.3 +cloudpickle==3.1.2 + # via joblib +contourpy==1.4.0 # via matplotlib -coverage[toml]==7.15.4 +coverage[toml]==7.16.2 # via pytest-cov -cryptography==50.0.0 +cryptography==50.0.2 # via # authlib # joserfc @@ -93,11 +92,11 @@ custodian==2025.12.14 # via mp-api (pyproject.toml) cycler==0.12.1 # via matplotlib -cyclopts==4.23.0 +cyclopts==5.1.1 # via fastmcp-slim deltalake==1.5.1 # via mp-api (pyproject.toml) -deprecated==1.3.1 +deprecated==3.0.0 # via deltalake distlib==0.4.3 # via virtualenv @@ -112,7 +111,7 @@ docutils==0.22.4 # via sphinx email-validator==2.3.0 # via pydantic -emmet-core[all]==0.87.2 +emmet-core[all]==0.87.3rc2 # via mp-api (pyproject.toml) exceptiongroup==1.3.1 # via fastmcp-slim @@ -120,17 +119,17 @@ execnet==2.1.2 # via pytest-xdist executing==2.2.1 # via stack-data -fastmcp==3.4.7 +fastmcp==4.0.10 # via mp-api (pyproject.toml) -fastmcp-slim[client,server]==3.4.7 +fastmcp-slim[client,server]==4.0.10 # via fastmcp -filelock==3.32.3 +filelock==4.0.9 # via # python-discovery # virtualenv filetype==1.2.0 # via mp-api (pyproject.toml) -flake8==7.3.0 +flake8==7.4.1 # via mp-api (pyproject.toml) flask==3.1.3 # via mp-api (pyproject.toml) @@ -138,38 +137,36 @@ flexcache==0.3 # via pint flexparser==0.4 # via pint -fonttools==4.63.0 +fonttools==4.66.1 # via matplotlib -fqdn==1.5.1 +fqdn==1.6.0 # via jsonschema -griffelib==2.2.0 +griffelib==2.3.0 # via fastmcp-slim h11==0.16.0 # via - # httpcore + # httpcore2 # uvicorn h5py==3.16.0 # via phonopy -httpcore==1.0.9 - # via httpx -httpx==0.28.1 +httpcore2==2.13.1 + # via httpx2 +httpx2==2.13.1 # via # fastmcp-slim # mcp -httpx-sse==0.4.3 - # via mcp -identify==2.6.19 +identify==2.6.20 # via pre-commit -idna==3.18 +idna==3.20 # via # anyio # email-validator - # httpx + # httpx2 # jsonschema # requests -imageio==2.37.4 +imageio==2.38.0 # via scikit-image -imagesize==2.0.0 +imagesize==2.0.1 # via sphinx importlib-resources==7.1.0 # via swagger-spec-validator @@ -177,7 +174,7 @@ inflect==7.5.0 # via emmet-core iniconfig==2.3.0 # via pytest -ipython==9.16.1 +ipython==9.17.1 # via mp-api (pyproject.toml) ipython-pygments-lexers==1.1.1 # via ipython @@ -205,11 +202,11 @@ jmespath==1.1.0 # via # boto3 # botocore -joblib==1.5.3 +joblib==1.6.0 # via # pymatgen-analysis-diffusion # pymatgen-core -joserfc==1.7.4 +joserfc==1.7.5 # via # authlib # fastmcp-slim @@ -232,19 +229,19 @@ jsonschema-specifications==2025.9.1 # via jsonschema keyring==25.7.0 # via py-key-value-aio -kiwisolver==1.5.0 +kiwisolver==1.5.1 # via matplotlib lark==1.3.1 # via rfc3987-syntax latexcodec==3.0.1 # via pybtex -lazy-loader==0.5 +lazy-loader==0.6 # via scikit-image -librt==0.15.0 +librt==0.16.0 # via mypy lobsterpy==0.6.1 # via emmet-core -lxml==6.1.1 +lxml==6.1.3 # via pymatgen-core markdown-it-py==4.2.0 # via rich @@ -253,7 +250,7 @@ markupsafe==3.0.3 # flask # jinja2 # werkzeug -matplotlib==3.11.1 +matplotlib==3.11.2 # via # ase # phonopy @@ -263,8 +260,12 @@ matplotlib-inline==0.2.2 # via ipython mccabe==0.7.0 # via flake8 -mcp==1.29.0 +mcp==2.2.0 # via fastmcp-slim +mcp-types==2.2.0 + # via + # fastmcp-slim + # mcp mdurl==0.1.2 # via markdown-it-py monotonic==1.6 @@ -284,25 +285,25 @@ mp-pyrho==0.5.1 # via pymatgen-analysis-defects mpmath==1.3.0 # via sympy -msgpack==1.2.1 +msgpack==1.2.3 # via # bravado # bravado-core -mypy==2.3.1 +mypy==2.4.0 # via mp-api (pyproject.toml) mypy-extensions==1.1.0 # via # mp-api (pyproject.toml) # mypy -narwhals==2.24.0 +narwhals==2.26.0 # via plotly -networkx==3.6.1 +networkx==3.7 # via # pymatgen-core # scikit-image -nodeenv==1.10.0 +nodeenv==1.11.0 # via pre-commit -numpy==2.5.2 +numpy==2.5.3 # via # ase # contourpy @@ -325,10 +326,12 @@ numpy==2.5.2 # spglib # symfc # tifffile -openapi-pydantic==0.5.1 - # via fastmcp-slim -opentelemetry-api==1.44.0 +openapi-pydantic==0.6.0 # via fastmcp-slim +opentelemetry-api==1.45.0 + # via + # fastmcp-slim + # mcp orjson==3.12.0 # via # mp-api (pyproject.toml) @@ -341,10 +344,12 @@ packaging==26.3 # plotly # pytest # scikit-image + # seekpath # sphinx + # virtualenv palettable==3.3.3 # via pymatgen-core -pandas==3.0.5 +pandas==3.0.6 # via # pymatgen-core # seaborn @@ -356,23 +361,23 @@ pathspec==1.1.1 # via mypy pexpect==4.9.0 # via ipython -phonopy==4.4.0 +phonopy==4.8.0 # via emmet-core -phonors==0.3.0 +phonors==0.6.0 # via phonopy pillow==12.3.0 # via # imageio # matplotlib # scikit-image -pint==0.25.3 +pint==0.26.1 # via mp-api (pyproject.toml) -platformdirs==4.11.3 +platformdirs==4.12.2 # via # fastmcp-slim # pint # virtualenv -plotly==6.9.0 +plotly==7.1.0 # via pymatgen-core pluggy==1.6.0 # via @@ -390,9 +395,9 @@ ptyprocess==0.7.0 # via pexpect pubchempy==1.0.5 # via emmet-core -pure-eval==0.2.3 +pure-eval==0.2.4 # via stack-data -py-key-value-aio[filetree,keyring,memory]==0.4.5 +py-key-value-aio[filetree,keyring,memory]==0.4.6 # via fastmcp-slim pyarrow==25.0.1 # via @@ -402,29 +407,29 @@ pybtex==0.26.1 # via emmet-core pycodcif==3.0.1 # via emmet-core -pycodestyle==2.14.0 +pycodestyle==2.15.0 # via # flake8 # mp-api (pyproject.toml) pycparser==3.0 # via cffi -pydantic[email]==2.13.4 +pydantic[email]==2.13.5 # via # emmet-core # fastmcp-slim # mcp + # mcp-types # openapi-pydantic # pydantic-settings # pymatgen-io-validation -pydantic-core==2.46.4 +pydantic-core==2.46.5 # via pydantic pydantic-settings==2.15.0 # via # emmet-core # fastmcp-slim - # mcp # pymatgen-io-validation -pyflakes==3.4.0 +pyflakes==4.0.1 # via flake8 pygments==2.21.0 # via @@ -438,7 +443,7 @@ pyisemail==2.0.1 # via mp-api (pyproject.toml) pyjwt[crypto]==2.15.1 # via mcp -pymatgen==2026.5.4 +pymatgen==2026.9.24 # via # emmet-core # lobsterpy @@ -451,20 +456,20 @@ pymatgen-analysis-alloys==0.0.9 # via emmet-core pymatgen-analysis-defects==2026.3.20 # via emmet-core -pymatgen-analysis-diffusion==2025.11.15 +pymatgen-analysis-diffusion==2026.9.23 # via emmet-core -pymatgen-core==2026.7.16 +pymatgen-core==2026.10.2 # via # emmet-core # pymatgen # pymatgen-io-validation pymatgen-io-validation==0.1.4 # via emmet-core -pymongo==4.17.0 +pymongo==4.18.2 # via # emmet-core # mp-api (pyproject.toml) -pyparsing==3.3.2 +pyparsing==3.3.3 # via # bibtexparser # matplotlib @@ -481,7 +486,7 @@ pytest-asyncio==1.4.0 # via mp-api (pyproject.toml) pytest-cov==7.1.0 # via mp-api (pyproject.toml) -pytest-mock==3.15.1 +pytest-mock==3.16.0 # via mp-api (pyproject.toml) pytest-xdist==3.8.0 # via mp-api (pyproject.toml) @@ -493,9 +498,9 @@ python-dateutil==2.9.0.post0 # bravado-core # matplotlib # pandas -python-discovery==1.5.2 +python-discovery==1.6.1 # via virtualenv -python-dotenv==1.2.3 +python-dotenv==1.2.4 # via # fastmcp-slim # pydantic-settings @@ -503,7 +508,7 @@ python-multipart==0.0.32 # via # fastmcp-slim # mcp -pytz==2026.3.post1 +pytz==2026.4 # via bravado-core pyyaml==6.0.3 # via @@ -529,7 +534,7 @@ requests==2.34.2 # pymatgen-io-validation # requests-futures # sphinx -requests-futures==1.0.2 +requests-futures==1.1.0 # via mp-api (pyproject.toml) rfc3339-validator==0.1.4 # via jsonschema @@ -542,7 +547,7 @@ rich==15.0.0 # cyclopts # fastmcp-slim # rich-rst -rich-rst==2.1.0 +rich-rst==2.2.0 # via cyclopts roman-numerals==4.1.0 # via sphinx @@ -558,7 +563,7 @@ s3transfer==0.19.2 # via boto3 scikit-image==0.26.0 # via pymatgen-analysis-defects -scipy==1.18.0 +scipy==1.18.1 # via # ase # pymatgen-core @@ -568,11 +573,11 @@ seaborn==0.13.2 # via pymatgen-analysis-diffusion secretstorage==3.5.0 # via keyring -seekpath==2.2.1 +seekpath==2.2.2 # via emmet-core shapely==2.1.2 # via pymatgen-analysis-alloys -simplejson==4.1.1 +simplejson==4.1.2 # via # bravado # bravado-core @@ -604,11 +609,11 @@ sphinxcontrib-qthelp==2.0.0 # via sphinx sphinxcontrib-serializinghtml==2.0.0 # via sphinx -sse-starlette==3.4.8 +sse-starlette==3.5.0 # via mcp stack-data==0.6.3 # via ipython -starlette==1.6.0 +starlette==1.7.0 # via # fastmcp-slim # mcp @@ -623,24 +628,29 @@ sympy==1.14.0 # via pymatgen-core tabulate==0.10.0 # via pymatgen-core -tifffile==2026.8.16 +tifffile==2026.9.20 # via scikit-image -tqdm==4.70.0 +tqdm==4.70.1 # via pymatgen-core traitlets==5.16.1 # via # ipython # matplotlib-inline +truststore==0.10.4 + # via + # httpcore2 + # httpx2 typeguard==4.6.0 # via inflect -types-requests==2.33.0.20260712 +types-requests==2.33.0.20260906 # via mp-api (pyproject.toml) types-setuptools==84.0.0.20260812 # via mp-api (pyproject.toml) -types-tqdm==4.70.0.20260805 +types-tqdm==4.70.0.20260906 # via mp-api (pyproject.toml) typing-extensions==4.16.0 # via + # anyio # ase # bravado # emmet-core @@ -648,6 +658,7 @@ typing-extensions==4.16.0 # flexcache # flexparser # mcp + # mcp-types # mp-api (pyproject.toml) # mypy # opentelemetry-api @@ -663,7 +674,7 @@ typing-inspection==0.4.4 # mcp # pydantic # pydantic-settings -tzdata==2026.3 +tzdata==2026.4 # via arrow uncalled-for==0.4.0 # via fastmcp-slim @@ -676,21 +687,21 @@ urllib3==2.8.0 # botocore # requests # types-requests -uvicorn==0.52.3 +uvicorn==0.54.0 # via # fastmcp-slim # mcp -virtualenv==21.7.13 +virtualenv==21.14.5 # via pre-commit -watchfiles==1.2.0 +watchfiles==1.3.0 # via fastmcp-slim -wcwidth==0.8.2 +wcwidth==0.9.1 # via prompt-toolkit webcolors==25.10.0 # via jsonschema -websockets==17.0.1 +websockets==17.1 # via fastmcp-slim -werkzeug==3.1.8 +werkzeug==3.1.9 # via flask -wrapt==2.3.0 +wrapt==2.5.0 # via deprecated diff --git a/requirements/requirements-ubuntu-latest_py3.14.txt b/requirements/requirements-ubuntu-latest_py3.14.txt index 17664e28..c1e1d61f 100644 --- a/requirements/requirements-ubuntu-latest_py3.14.txt +++ b/requirements/requirements-ubuntu-latest_py3.14.txt @@ -2,39 +2,41 @@ # This file is autogenerated by pip-compile with Python 3.14 # by the following command: # -# pip-compile --output-file=requirements/requirements-ubuntu-latest_py3.14.txt pyproject.toml +# pip-compile --no-index --output-file=requirements/requirements-ubuntu-latest_py3.14.txt pyproject.toml # annotated-types==0.8.0 # via pydantic -arro3-core==0.8.1 +arro3-core==0.9.0 # via deltalake bibtexparser==1.4.4 # via pymatgen-core -blake3==1.0.9 +blake3==1.0.10 # via emmet-core -boto3==1.43.73 +boto3==1.43.108 # via mp-api (pyproject.toml) -botocore==1.43.73 +botocore==1.43.108 # via # boto3 # s3transfer certifi==2026.7.22 # via requests -charset-normalizer==3.5.1 +charset-normalizer==3.5.2 # via requests -contourpy==1.3.3 +cloudpickle==3.1.2 + # via joblib +contourpy==1.4.0 # via matplotlib cycler==0.12.1 # via matplotlib deltalake==1.5.1 # via mp-api (pyproject.toml) -deprecated==1.3.1 +deprecated==3.0.0 # via deltalake -emmet-core==0.87.2 +emmet-core==0.87.3rc2 # via mp-api (pyproject.toml) -fonttools==4.63.0 +fonttools==4.66.1 # via matplotlib -idna==3.18 +idna==3.20 # via requests inflect==7.5.0 # via emmet-core @@ -42,15 +44,15 @@ jmespath==1.1.0 # via # boto3 # botocore -joblib==1.5.3 +joblib==1.6.0 # via pymatgen-core -kiwisolver==1.5.0 +kiwisolver==1.5.1 # via matplotlib latexcodec==3.0.1 # via pybtex -lxml==6.1.1 +lxml==6.1.3 # via pymatgen-core -matplotlib==3.11.1 +matplotlib==3.11.2 # via pymatgen-core monty==2026.7.16 # via @@ -61,11 +63,11 @@ more-itertools==11.1.0 # via inflect mpmath==1.3.0 # via sympy -narwhals==2.24.0 +narwhals==2.26.0 # via plotly -networkx==3.6.1 +networkx==3.7 # via pymatgen-core -numpy==2.5.2 +numpy==2.5.3 # via # contourpy # matplotlib @@ -85,11 +87,11 @@ packaging==26.3 # plotly palettable==3.3.3 # via pymatgen-core -pandas==3.0.5 +pandas==3.0.6 # via pymatgen-core pillow==12.3.0 # via matplotlib -plotly==6.9.0 +plotly==7.1.0 # via pymatgen-core pubchempy==1.0.5 # via emmet-core @@ -97,28 +99,29 @@ pyarrow==25.0.1 # via mp-api (pyproject.toml) pybtex==0.26.1 # via emmet-core -pydantic==2.13.4 +pydantic==2.13.5 # via # emmet-core # pydantic-settings # pymatgen-io-validation -pydantic-core==2.46.4 +pydantic-core==2.46.5 # via pydantic pydantic-settings==2.15.0 # via # emmet-core # pymatgen-io-validation -pymatgen==2026.5.4 +pymatgen==2026.9.24 # via # emmet-core # mp-api (pyproject.toml) -pymatgen-core==2026.8.13 +pymatgen-core==2026.10.2 # via + # emmet-core # pymatgen # pymatgen-io-validation pymatgen-io-validation==0.1.4 # via emmet-core -pyparsing==3.3.2 +pyparsing==3.3.3 # via # bibtexparser # matplotlib @@ -127,7 +130,7 @@ python-dateutil==2.9.0.post0 # botocore # matplotlib # pandas -python-dotenv==1.2.3 +python-dotenv==1.2.4 # via pydantic-settings pyyaml==6.0.3 # via pybtex @@ -140,7 +143,7 @@ ruamel-yaml==0.19.1 # via monty s3transfer==0.19.2 # via boto3 -scipy==1.18.0 +scipy==1.18.1 # via pymatgen-core six==1.17.0 # via python-dateutil @@ -150,7 +153,7 @@ sympy==1.14.0 # via pymatgen-core tabulate==0.10.0 # via pymatgen-core -tqdm==4.70.0 +tqdm==4.70.1 # via pymatgen-core typeguard==4.6.0 # via inflect @@ -172,5 +175,5 @@ urllib3==2.8.0 # via # botocore # requests -wrapt==2.3.0 +wrapt==2.5.0 # via deprecated diff --git a/requirements/requirements-ubuntu-latest_py3.14_extras.txt b/requirements/requirements-ubuntu-latest_py3.14_extras.txt index aecebaea..5db9a741 100644 --- a/requirements/requirements-ubuntu-latest_py3.14_extras.txt +++ b/requirements/requirements-ubuntu-latest_py3.14_extras.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.14 # by the following command: # -# pip-compile --all-extras --output-file=requirements/requirements-ubuntu-latest_py3.14_extras.txt pyproject.toml +# pip-compile --all-extras --no-index --output-file=requirements/requirements-ubuntu-latest_py3.14_extras.txt pyproject.toml # aiofile==3.12.3 # via py-key-value-aio @@ -10,21 +10,21 @@ alabaster==1.0.0 # via sphinx annotated-types==0.8.0 # via pydantic -anyio==4.14.2 +anyio==4.15.1 # via - # httpx + # httpx2 # mcp # py-key-value-aio # sse-starlette # starlette # watchfiles -arro3-core==0.8.1 +arro3-core==0.9.0 # via deltalake arrow==1.4.0 # via isoduration ase==3.29.0 # via pymatgen-analysis-diffusion -ast-serialize==0.8.0 +ast-serialize==0.12.0 # via mypy asttokens==3.0.2 # via stack-data @@ -34,7 +34,7 @@ attrs==26.1.0 # jsonschema # jsonschema-path # referencing -authlib==1.7.2 +authlib==1.8.0 # via fastmcp-slim babel==2.18.0 # via sphinx @@ -42,15 +42,15 @@ beartype==0.22.9 # via py-key-value-aio bibtexparser==1.4.4 # via pymatgen-core -blake3==1.0.9 +blake3==1.0.10 # via emmet-core blinker==1.9.0 # via flask -boltons==26.1.0 +boltons==26.2.0 # via mp-api (pyproject.toml) -boto3==1.43.73 +boto3==1.43.108 # via mp-api (pyproject.toml) -botocore==1.43.73 +botocore==1.43.108 # via # boto3 # s3transfer @@ -58,32 +58,31 @@ bravado==12.0.1 # via mp-api (pyproject.toml) bravado-core==6.4.1 # via bravado -cachetools==7.1.7 +cachetools==7.2.0 # via # mp-api (pyproject.toml) # py-key-value-aio -caio==0.12.2 +caio==0.12.9 # via aiofile certifi==2026.7.22 - # via - # httpcore - # httpx - # requests + # via requests cffi==2.1.1 # via cryptography cfgv==3.5.0 # via pre-commit -charset-normalizer==3.5.1 +charset-normalizer==3.5.2 # via requests -click==8.4.2 +click==8.5.0 # via # flask # uvicorn -contourpy==1.3.3 +cloudpickle==3.1.2 + # via joblib +contourpy==1.4.0 # via matplotlib -coverage[toml]==7.15.4 +coverage[toml]==7.16.2 # via pytest-cov -cryptography==50.0.0 +cryptography==50.0.2 # via # authlib # joserfc @@ -93,11 +92,11 @@ custodian==2025.12.14 # via mp-api (pyproject.toml) cycler==0.12.1 # via matplotlib -cyclopts==4.23.0 +cyclopts==5.1.1 # via fastmcp-slim deltalake==1.5.1 # via mp-api (pyproject.toml) -deprecated==1.3.1 +deprecated==3.0.0 # via deltalake distlib==0.4.3 # via virtualenv @@ -112,7 +111,7 @@ docutils==0.22.4 # via sphinx email-validator==2.3.0 # via pydantic -emmet-core[all]==0.87.2 +emmet-core[all]==0.87.3rc2 # via mp-api (pyproject.toml) exceptiongroup==1.3.1 # via fastmcp-slim @@ -120,17 +119,17 @@ execnet==2.1.2 # via pytest-xdist executing==2.2.1 # via stack-data -fastmcp==3.4.7 +fastmcp==4.0.10 # via mp-api (pyproject.toml) -fastmcp-slim[client,server]==3.4.7 +fastmcp-slim[client,server]==4.0.10 # via fastmcp -filelock==3.32.3 +filelock==4.0.9 # via # python-discovery # virtualenv filetype==1.2.0 # via mp-api (pyproject.toml) -flake8==7.3.0 +flake8==7.4.1 # via mp-api (pyproject.toml) flask==3.1.3 # via mp-api (pyproject.toml) @@ -138,38 +137,36 @@ flexcache==0.3 # via pint flexparser==0.4 # via pint -fonttools==4.63.0 +fonttools==4.66.1 # via matplotlib -fqdn==1.5.1 +fqdn==1.6.0 # via jsonschema -griffelib==2.2.0 +griffelib==2.3.0 # via fastmcp-slim h11==0.16.0 # via - # httpcore + # httpcore2 # uvicorn h5py==3.16.0 # via phonopy -httpcore==1.0.9 - # via httpx -httpx==0.28.1 +httpcore2==2.13.1 + # via httpx2 +httpx2==2.13.1 # via # fastmcp-slim # mcp -httpx-sse==0.4.3 - # via mcp -identify==2.6.19 +identify==2.6.20 # via pre-commit -idna==3.18 +idna==3.20 # via # anyio # email-validator - # httpx + # httpx2 # jsonschema # requests -imageio==2.37.4 +imageio==2.38.0 # via scikit-image -imagesize==2.0.0 +imagesize==2.0.1 # via sphinx importlib-resources==7.1.0 # via swagger-spec-validator @@ -177,7 +174,7 @@ inflect==7.5.0 # via emmet-core iniconfig==2.3.0 # via pytest -ipython==9.16.1 +ipython==9.17.1 # via mp-api (pyproject.toml) ipython-pygments-lexers==1.1.1 # via ipython @@ -205,11 +202,11 @@ jmespath==1.1.0 # via # boto3 # botocore -joblib==1.5.3 +joblib==1.6.0 # via # pymatgen-analysis-diffusion # pymatgen-core -joserfc==1.7.4 +joserfc==1.7.5 # via # authlib # fastmcp-slim @@ -232,19 +229,19 @@ jsonschema-specifications==2025.9.1 # via jsonschema keyring==25.7.0 # via py-key-value-aio -kiwisolver==1.5.0 +kiwisolver==1.5.1 # via matplotlib lark==1.3.1 # via rfc3987-syntax latexcodec==3.0.1 # via pybtex -lazy-loader==0.5 +lazy-loader==0.6 # via scikit-image -librt==0.15.0 +librt==0.16.0 # via mypy lobsterpy==0.6.1 # via emmet-core -lxml==6.1.1 +lxml==6.1.3 # via pymatgen-core markdown-it-py==4.2.0 # via rich @@ -253,7 +250,7 @@ markupsafe==3.0.3 # flask # jinja2 # werkzeug -matplotlib==3.11.1 +matplotlib==3.11.2 # via # ase # phonopy @@ -263,8 +260,12 @@ matplotlib-inline==0.2.2 # via ipython mccabe==0.7.0 # via flake8 -mcp==1.29.0 +mcp==2.2.0 # via fastmcp-slim +mcp-types==2.2.0 + # via + # fastmcp-slim + # mcp mdurl==0.1.2 # via markdown-it-py monotonic==1.6 @@ -284,25 +285,25 @@ mp-pyrho==0.5.1 # via pymatgen-analysis-defects mpmath==1.3.0 # via sympy -msgpack==1.2.1 +msgpack==1.2.3 # via # bravado # bravado-core -mypy==2.3.1 +mypy==2.4.0 # via mp-api (pyproject.toml) mypy-extensions==1.1.0 # via # mp-api (pyproject.toml) # mypy -narwhals==2.24.0 +narwhals==2.26.0 # via plotly -networkx==3.6.1 +networkx==3.7 # via # pymatgen-core # scikit-image -nodeenv==1.10.0 +nodeenv==1.11.0 # via pre-commit -numpy==2.5.2 +numpy==2.5.3 # via # ase # contourpy @@ -325,10 +326,12 @@ numpy==2.5.2 # spglib # symfc # tifffile -openapi-pydantic==0.5.1 - # via fastmcp-slim -opentelemetry-api==1.44.0 +openapi-pydantic==0.6.0 # via fastmcp-slim +opentelemetry-api==1.45.0 + # via + # fastmcp-slim + # mcp orjson==3.12.0 # via # mp-api (pyproject.toml) @@ -341,10 +344,12 @@ packaging==26.3 # plotly # pytest # scikit-image + # seekpath # sphinx + # virtualenv palettable==3.3.3 # via pymatgen-core -pandas==3.0.5 +pandas==3.0.6 # via # pymatgen-core # seaborn @@ -356,23 +361,23 @@ pathspec==1.1.1 # via mypy pexpect==4.9.0 # via ipython -phonopy==4.4.0 +phonopy==4.8.0 # via emmet-core -phonors==0.3.0 +phonors==0.6.0 # via phonopy pillow==12.3.0 # via # imageio # matplotlib # scikit-image -pint==0.25.3 +pint==0.26.1 # via mp-api (pyproject.toml) -platformdirs==4.11.3 +platformdirs==4.12.2 # via # fastmcp-slim # pint # virtualenv -plotly==6.9.0 +plotly==7.1.0 # via pymatgen-core pluggy==1.6.0 # via @@ -390,9 +395,9 @@ ptyprocess==0.7.0 # via pexpect pubchempy==1.0.5 # via emmet-core -pure-eval==0.2.3 +pure-eval==0.2.4 # via stack-data -py-key-value-aio[filetree,keyring,memory]==0.4.5 +py-key-value-aio[filetree,keyring,memory]==0.4.6 # via fastmcp-slim pyarrow==25.0.1 # via @@ -402,29 +407,29 @@ pybtex==0.26.1 # via emmet-core pycodcif==3.0.1 # via emmet-core -pycodestyle==2.14.0 +pycodestyle==2.15.0 # via # flake8 # mp-api (pyproject.toml) pycparser==3.0 # via cffi -pydantic[email]==2.13.4 +pydantic[email]==2.13.5 # via # emmet-core # fastmcp-slim # mcp + # mcp-types # openapi-pydantic # pydantic-settings # pymatgen-io-validation -pydantic-core==2.46.4 +pydantic-core==2.46.5 # via pydantic pydantic-settings==2.15.0 # via # emmet-core # fastmcp-slim - # mcp # pymatgen-io-validation -pyflakes==3.4.0 +pyflakes==4.0.1 # via flake8 pygments==2.21.0 # via @@ -438,7 +443,7 @@ pyisemail==2.0.1 # via mp-api (pyproject.toml) pyjwt[crypto]==2.15.1 # via mcp -pymatgen==2026.5.4 +pymatgen==2026.9.24 # via # emmet-core # lobsterpy @@ -451,20 +456,20 @@ pymatgen-analysis-alloys==0.0.9 # via emmet-core pymatgen-analysis-defects==2026.3.20 # via emmet-core -pymatgen-analysis-diffusion==2025.11.15 +pymatgen-analysis-diffusion==2026.9.23 # via emmet-core -pymatgen-core==2026.7.16 +pymatgen-core==2026.10.2 # via # emmet-core # pymatgen # pymatgen-io-validation pymatgen-io-validation==0.1.4 # via emmet-core -pymongo==4.17.0 +pymongo==4.18.2 # via # emmet-core # mp-api (pyproject.toml) -pyparsing==3.3.2 +pyparsing==3.3.3 # via # bibtexparser # matplotlib @@ -481,7 +486,7 @@ pytest-asyncio==1.4.0 # via mp-api (pyproject.toml) pytest-cov==7.1.0 # via mp-api (pyproject.toml) -pytest-mock==3.15.1 +pytest-mock==3.16.0 # via mp-api (pyproject.toml) pytest-xdist==3.8.0 # via mp-api (pyproject.toml) @@ -493,9 +498,9 @@ python-dateutil==2.9.0.post0 # bravado-core # matplotlib # pandas -python-discovery==1.5.2 +python-discovery==1.6.1 # via virtualenv -python-dotenv==1.2.3 +python-dotenv==1.2.4 # via # fastmcp-slim # pydantic-settings @@ -503,7 +508,7 @@ python-multipart==0.0.32 # via # fastmcp-slim # mcp -pytz==2026.3.post1 +pytz==2026.4 # via bravado-core pyyaml==6.0.3 # via @@ -529,7 +534,7 @@ requests==2.34.2 # pymatgen-io-validation # requests-futures # sphinx -requests-futures==1.0.2 +requests-futures==1.1.0 # via mp-api (pyproject.toml) rfc3339-validator==0.1.4 # via jsonschema @@ -542,7 +547,7 @@ rich==15.0.0 # cyclopts # fastmcp-slim # rich-rst -rich-rst==2.1.0 +rich-rst==2.2.0 # via cyclopts roman-numerals==4.1.0 # via sphinx @@ -558,7 +563,7 @@ s3transfer==0.19.2 # via boto3 scikit-image==0.26.0 # via pymatgen-analysis-defects -scipy==1.18.0 +scipy==1.18.1 # via # ase # pymatgen-core @@ -568,11 +573,11 @@ seaborn==0.13.2 # via pymatgen-analysis-diffusion secretstorage==3.5.0 # via keyring -seekpath==2.2.1 +seekpath==2.2.2 # via emmet-core shapely==2.1.2 # via pymatgen-analysis-alloys -simplejson==4.1.1 +simplejson==4.1.2 # via # bravado # bravado-core @@ -604,11 +609,11 @@ sphinxcontrib-qthelp==2.0.0 # via sphinx sphinxcontrib-serializinghtml==2.0.0 # via sphinx -sse-starlette==3.4.8 +sse-starlette==3.5.0 # via mcp stack-data==0.6.3 # via ipython -starlette==1.6.0 +starlette==1.7.0 # via # fastmcp-slim # mcp @@ -623,24 +628,29 @@ sympy==1.14.0 # via pymatgen-core tabulate==0.10.0 # via pymatgen-core -tifffile==2026.8.16 +tifffile==2026.9.20 # via scikit-image -tqdm==4.70.0 +tqdm==4.70.1 # via pymatgen-core traitlets==5.16.1 # via # ipython # matplotlib-inline +truststore==0.10.4 + # via + # httpcore2 + # httpx2 typeguard==4.6.0 # via inflect -types-requests==2.33.0.20260712 +types-requests==2.33.0.20260906 # via mp-api (pyproject.toml) types-setuptools==84.0.0.20260812 # via mp-api (pyproject.toml) -types-tqdm==4.70.0.20260805 +types-tqdm==4.70.0.20260906 # via mp-api (pyproject.toml) typing-extensions==4.16.0 # via + # anyio # ase # bravado # emmet-core @@ -648,6 +658,7 @@ typing-extensions==4.16.0 # flexcache # flexparser # mcp + # mcp-types # mp-api (pyproject.toml) # mypy # opentelemetry-api @@ -663,7 +674,7 @@ typing-inspection==0.4.4 # mcp # pydantic # pydantic-settings -tzdata==2026.3 +tzdata==2026.4 # via arrow uncalled-for==0.4.0 # via fastmcp-slim @@ -676,21 +687,21 @@ urllib3==2.8.0 # botocore # requests # types-requests -uvicorn==0.52.3 +uvicorn==0.54.0 # via # fastmcp-slim # mcp -virtualenv==21.7.13 +virtualenv==21.14.5 # via pre-commit -watchfiles==1.2.0 +watchfiles==1.3.0 # via fastmcp-slim -wcwidth==0.8.2 +wcwidth==0.9.1 # via prompt-toolkit webcolors==25.10.0 # via jsonschema -websockets==17.0.1 +websockets==17.1 # via fastmcp-slim -werkzeug==3.1.8 +werkzeug==3.1.9 # via flask -wrapt==2.3.0 +wrapt==2.5.0 # via deprecated diff --git a/tests/client/core/test_delta.py b/tests/client/core/test_delta.py new file mode 100644 index 00000000..a6f201d8 --- /dev/null +++ b/tests/client/core/test_delta.py @@ -0,0 +1,505 @@ +"""Tests for DeltaCatalog snapshot caching and refresh-on-missing-object retry. + +deltalake's DeltaTable and QueryBuilder are replaced with in-memory fakes, so +these tests need no network access, API key, or files on disk. +""" + +import threading +import warnings + +import pyarrow as pa +import pytest +from deltalake.exceptions import DeltaError + +import mp_api.client.core.delta as delta_mod +from mp_api.client.core.client import BaseRester, QueryBuilderWithCache, _Rester +from mp_api.client.core.delta import DeltaCatalog, _is_missing_object +from mp_api.client.core.exceptions import MPRestError, MPRestWarning + +REAL_ERROR = ( + "Failed to fetch metadata for file Users/tsmathis/mp_datasets/build/collections/" + "materials/version=2026-04-13/group-4-part-0.zstd.parquet: Parquet error: External: " + "Object at location /Users/tsmathis/mp_datasets/build/collections/materials/" + "version=2026-04-13/group-4-part-0.zstd.parquet not found: No such file or " + "directory (os error 2)" +) + + +class FakeDeltaTable: + """Stand-in for deltalake.DeltaTable; each construction is a new 'snapshot'.""" + + loads: list[str] = [] + + def __init__(self, uri, storage_options=None, **kwargs): + self.table_uri = uri + self._storage_options = storage_options + FakeDeltaTable.loads.append(uri) + self.snapshot = FakeDeltaTable.loads.count(uri) + + def version(self): + return self.snapshot + + +class FakeReader: + """Arrow C-stream producer, like the arro3 reader returned by QueryBuilder.execute.""" + + def __init__(self, result): + self._result = result + + def __arrow_c_stream__(self, requested_schema=None): + if isinstance(self._result, Exception): + # error surfacing while batches are read + raise self._result + return self._result.__arrow_c_stream__(requested_schema) + + +class FakeQueryBuilder: + """Stand-in for deltalake.QueryBuilder that can be told to fail.""" + + instances: list["FakeQueryBuilder"] = [] + # Queue of exceptions raised by successive execute() calls (any builder) + failures: list[Exception] = [] + # Raise failures while reading batches instead of from execute() + fail_on_read: bool = False + + def __init__(self): + self.registered: dict[str, FakeDeltaTable] = {} + FakeQueryBuilder.instances.append(self) + + def register(self, table_name, delta_table): + if table_name in self.registered: + raise DeltaError(f"The table {table_name} already exists") + self.registered[table_name] = delta_table + return self + + def execute(self, sql): + result = pa.table({"sql": [sql]}) + if FakeQueryBuilder.failures: + exc = FakeQueryBuilder.failures.pop(0) + if not FakeQueryBuilder.fail_on_read: + raise exc + result = exc + return FakeReader(result) + + +@pytest.fixture(autouse=True) +def fake_deltalake(monkeypatch): + FakeDeltaTable.loads = [] + FakeQueryBuilder.instances = [] + FakeQueryBuilder.failures = [] + FakeQueryBuilder.fail_on_read = False + monkeypatch.setattr(delta_mod, "DeltaTable", FakeDeltaTable) + monkeypatch.setattr(delta_mod, "QueryBuilder", FakeQueryBuilder) + # QueryBuilderWithCache subclasses the real QueryBuilder; skip its native init + monkeypatch.setattr( + "deltalake.query.QueryBuilder.__init__", lambda self: None, raising=True + ) + + +@pytest.fixture +def catalog(): + cat = DeltaCatalog() + cat.get_table("s3a://bucket/a/", "a") + cat.get_table("s3a://bucket/b/", "b") + return cat + + +# --- _is_missing_object -------------------------------------------------------- + + +@pytest.mark.parametrize( + "exc", + [ + DeltaError(REAL_ERROR), + OSError(REAL_ERROR), + FileNotFoundError("gone"), + DeltaError("Generic S3 error: ... status code: 404 Not Found"), + DeltaError("Object Store error: NotFound { path: ... }"), + ], +) +def test_is_missing_object_matches(exc): + assert _is_missing_object(exc) + + +@pytest.mark.parametrize( + "exc", + [ + DeltaError('SQL error: ParserError("Expected: an expression")'), + DeltaError("Schema error: No field named foo"), + OSError("Generic S3 error: request timed out"), + ], +) +def test_is_missing_object_ignores_other_errors(exc): + assert not _is_missing_object(exc) + + +# --- caching ------------------------------------------------------------------- + + +def test_get_table_caches_by_uri(catalog): + label, table = catalog.get_table("s3a://bucket/a/", "a") + assert label == "a" + assert FakeDeltaTable.loads.count("s3a://bucket/a/") == 1 + assert catalog.labels == ["a", "b"] + assert "a" in catalog and len(catalog) == 2 + assert catalog.tables["a"] is table + + +def test_get_table_returns_stored_label_for_known_uri(catalog): + label, _ = catalog.get_table("s3a://bucket/a/", "other_name") + assert label == "a" + assert "other_name" not in catalog + + +def test_label_collision_with_different_uri_raises(catalog): + with pytest.raises(ValueError, match="already registered"): + catalog.get_table("s3a://bucket/c/", "a") + + +def test_get_table_refresh_reloads_only_that_table(catalog): + _, old_b = catalog.get_table("s3a://bucket/b/", "b") + old_qb = catalog._qb + + _, new_a = catalog.get_table("s3a://bucket/a/", "a", refresh=True) + + assert new_a.snapshot == 2 + assert catalog._qb is not old_qb + # new builder has every table, with untouched tables carried over as-is + assert catalog._qb.registered == {"a": new_a, "b": old_b} + assert FakeDeltaTable.loads.count("s3a://bucket/b/") == 1 + + +# --- execute / retry ----------------------------------------------------------- + + +def test_execute_success_no_refresh(catalog): + result = catalog.execute("SELECT * FROM a", label="a") + assert result["sql"].to_pylist() == ["SELECT * FROM a"] + assert len(FakeQueryBuilder.instances) == 1 + + +@pytest.mark.parametrize("fail_on_read", [False, True]) +def test_missing_object_refreshes_named_table_and_retries(catalog, fail_on_read): + FakeQueryBuilder.fail_on_read = fail_on_read + FakeQueryBuilder.failures = [DeltaError(REAL_ERROR)] + _, old_b = catalog.get_table("s3a://bucket/b/", "b") + + result = catalog.execute("SELECT * FROM a", label="a") + + assert result.num_rows == 1 + assert FakeDeltaTable.loads.count("s3a://bucket/a/") == 2 + assert FakeDeltaTable.loads.count("s3a://bucket/b/") == 1 + assert catalog.tables["b"] is old_b + assert catalog.tables["a"].snapshot == 2 + assert set(catalog._qb.registered) == {"a", "b"} + + +def test_non_missing_error_is_not_retried(catalog): + FakeQueryBuilder.failures = [DeltaError("SQL error: bad syntax")] + with pytest.raises(DeltaError, match="bad syntax"): + catalog.execute("SELECT * FROM a", label="a") + assert FakeDeltaTable.loads.count("s3a://bucket/a/") == 1 + + +def test_no_label_is_not_retried(catalog): + FakeQueryBuilder.failures = [DeltaError(REAL_ERROR)] + with pytest.raises(DeltaError): + catalog.execute("SELECT * FROM a") + assert FakeDeltaTable.loads.count("s3a://bucket/a/") == 1 + + +def test_second_failure_raises_with_note(catalog): + FakeQueryBuilder.failures = [DeltaError(REAL_ERROR), DeltaError(REAL_ERROR)] + with pytest.raises(DeltaError) as excinfo: + catalog.execute("SELECT * FROM a", label="a") + assert any("after refreshing" in n for n in excinfo.value.__notes__) + # exactly one refresh + assert FakeDeltaTable.loads.count("s3a://bucket/a/") == 2 + + +def test_concurrent_refresh_is_not_repeated(catalog): + """A refresh by another thread between failure and retry isn't redone.""" + entry = catalog._entries["s3a://bucket/a/"] + seen = entry.generation + catalog.get_table("s3a://bucket/a/", "a", refresh=True) # "other thread" + loads_before = len(FakeDeltaTable.loads) + + qb = catalog._refresh(entry, seen) + + assert len(FakeDeltaTable.loads) == loads_before + assert qb is catalog._qb + + +# --- runner hook --------------------------------------------------------------- + + +class RecordingRunner: + def __init__(self): + self.calls = [] + + def __call__(self, fn): + name = getattr(getattr(fn, "func", fn), "__name__", repr(fn)) + self.calls.append(name) + return fn() + + +def test_default_runner_calls_directly(): + cat = DeltaCatalog() + assert cat.runner(lambda: 42) == 42 + + +def test_runner_wraps_loads_queries_and_reloads(): + runner = RecordingRunner() + cat = DeltaCatalog(runner=runner) + + cat.get_table("s3a://bucket/a/", "a") + assert runner.calls == ["FakeDeltaTable"] + + cat.execute("SELECT * FROM a", label="a") + assert runner.calls[-1] == "_run" + + runner.calls.clear() + FakeQueryBuilder.failures = [DeltaError(REAL_ERROR)] + assert cat.execute("SELECT * FROM a", label="a").num_rows == 1 + # failed query, reload, retried query + assert runner.calls == ["_run", "FakeDeltaTable", "_run"] + + runner.calls.clear() + cat.get_table("s3a://bucket/a/", "a", refresh=True) + assert runner.calls == ["FakeDeltaTable"] + + # cache hit: no native work + runner.calls.clear() + cat.get_table("s3a://bucket/a/", "a") + assert runner.calls == [] + + +def test_runner_exceptions_propagate(): + def runner(fn): + return fn() + + cat = DeltaCatalog(runner=runner) + cat.get_table("s3a://bucket/a/", "a") + FakeQueryBuilder.failures = [DeltaError("SQL error: bad syntax")] + with pytest.raises(DeltaError, match="bad syntax"): + cat.execute("SELECT * FROM a", label="a") + + +def test_threadpool_runner(): + from concurrent.futures import ThreadPoolExecutor + + main = threading.get_ident() + seen = [] + + with ThreadPoolExecutor(2) as pool: + + def runner(fn): + def wrapped(): + seen.append(threading.get_ident()) + return fn() + + return pool.submit(wrapped).result() + + cat = DeltaCatalog(runner=runner) + cat.get_table("s3a://bucket/a/", "a") + FakeQueryBuilder.failures = [DeltaError(REAL_ERROR)] + assert cat.execute("SELECT * FROM a", label="a").num_rows == 1 + + assert seen and main not in seen + + +# --- concurrency --------------------------------------------------------------- + + +class BlockingLoads: + """Patch FakeDeltaTable so loads of one URI block until released.""" + + def __init__(self, monkeypatch, uri): + self.uri = uri + self.started = threading.Event() + self.release = threading.Event() + original = FakeDeltaTable.__init__ + + def init(table, table_uri, storage_options=None, **kwargs): + if table_uri == self.uri: + self.started.set() + assert self.release.wait(5), "test timed out waiting for release" + original(table, table_uri, storage_options=storage_options, **kwargs) + + monkeypatch.setattr(FakeDeltaTable, "__init__", init) + + +def _in_thread(fn, *args, **kwargs): + result = {} + + def target(): + try: + result["value"] = fn(*args, **kwargs) + except Exception as exc: # pragma: no cover - surfaced via assert + result["error"] = exc + + th = threading.Thread(target=target, daemon=True) + th.start() + return th, result + + +def test_reload_does_not_block_queries_on_other_tables(catalog, monkeypatch): + blocker = BlockingLoads(monkeypatch, "s3a://bucket/a/") + + th, res = _in_thread(catalog.get_table, "s3a://bucket/a/", "a", refresh=True) + assert blocker.started.wait(5) + + # While a's reload is stuck in the network call, b is still queryable + # and a new table can be added. + assert catalog.execute("SELECT * FROM b", label="b").num_rows == 1 + catalog.get_table("s3a://bucket/c/", "c") + + blocker.release.set() + th.join(5) + assert "error" not in res + # reload swapped in a builder holding all three tables + assert set(catalog._qb.registered) == {"a", "b", "c"} + assert catalog.tables["a"].snapshot == 2 + + +def test_concurrent_first_loads_of_same_uri_load_once(monkeypatch): + cat = DeltaCatalog() + blocker = BlockingLoads(monkeypatch, "s3a://bucket/a/") + + t1, r1 = _in_thread(cat.get_table, "s3a://bucket/a/", "a") + assert blocker.started.wait(5) + t2, r2 = _in_thread(cat.get_table, "s3a://bucket/a/", "a") + + blocker.release.set() + t1.join(5) + t2.join(5) + + assert FakeDeltaTable.loads.count("s3a://bucket/a/") == 1 + assert r1["value"][1] is r2["value"][1] + + +def test_concurrent_refreshes_of_same_table_reload_once(catalog, monkeypatch): + blocker = BlockingLoads(monkeypatch, "s3a://bucket/a/") + FakeQueryBuilder.failures = [DeltaError(REAL_ERROR), DeltaError(REAL_ERROR)] + + t1, r1 = _in_thread(catalog.execute, "SELECT * FROM a", label="a") + assert blocker.started.wait(5) + t2, r2 = _in_thread(catalog.execute, "SELECT * FROM a", label="a") + + blocker.release.set() + t1.join(5) + t2.join(5) + + assert "error" not in r1 and "error" not in r2 + # initial load + exactly one reload + assert FakeDeltaTable.loads.count("s3a://bucket/a/") == 2 + + +def test_failed_load_is_not_registered(monkeypatch): + cat = DeltaCatalog() + + def boom(table, table_uri, **kwargs): + raise OSError("connection refused") + + monkeypatch.setattr(FakeDeltaTable, "__init__", boom) + with pytest.raises(OSError): + cat.get_table("s3a://bucket/a/", "a") + assert "a" not in cat and len(cat) == 0 + + +def test_failed_reload_keeps_old_snapshot(catalog, monkeypatch): + old_a = catalog.tables["a"] + old_qb = catalog._qb + + def boom(table, table_uri, **kwargs): + raise OSError("connection refused") + + monkeypatch.setattr(FakeDeltaTable, "__init__", boom) + FakeQueryBuilder.failures = [DeltaError(REAL_ERROR)] + with pytest.raises(OSError): + catalog.execute("SELECT * FROM a", label="a") + assert catalog.tables["a"] is old_a + assert catalog._qb is old_qb + + +# --- rester integration -------------------------------------------------------- + + +@pytest.fixture +def no_heartbeat(monkeypatch): + monkeypatch.setattr( + BaseRester, "_get_heartbeat_info", staticmethod(lambda endpoint: ("v", [])) + ) + + +def test_rester_creates_catalog_lazily(): + rester = _Rester(api_key="a" * 32) + assert rester._delta_catalog is None + assert isinstance(rester.delta_catalog, DeltaCatalog) + assert rester.delta_catalog is rester.delta_catalog + + +def test_rester_uses_provided_catalog(): + cat = DeltaCatalog() + assert _Rester(api_key="a" * 32, delta_catalog=cat).delta_catalog is cat + + +def test_query_builder_with_cache_is_deprecated_but_shared(): + with pytest.warns(DeprecationWarning, match="QueryBuilderWithCache"): + qb = QueryBuilderWithCache() + r1 = _Rester(api_key="a" * 32, query_builder=qb) + r2 = _Rester(api_key="a" * 32, query_builder=qb) + assert r1.delta_catalog is r2.delta_catalog is qb.catalog + + +def test_query_builder_with_cache_register_and_introspect(): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + qb = QueryBuilderWithCache() + table = FakeDeltaTable("s3a://bucket/x/") + assert qb.register("x", table) is qb + assert qb._delta_tables == {"x": table} + assert "x" in qb.catalog + + +def test_query_builder_property_is_deprecated(): + rester = _Rester(api_key="a" * 32) + with pytest.warns(DeprecationWarning, match="delta_catalog"): + qb = rester.query_builder + assert qb.catalog is rester.delta_catalog + + +def test_sub_resters_share_parent_catalog(no_heartbeat): + from mp_api.client.routes.materials.materials import MaterialsRester + + cat = DeltaCatalog() + materials = MaterialsRester(api_key="a" * 32, delta_catalog=cat) + assert materials.thermo.delta_catalog is cat + assert materials.tasks.delta_catalog is cat + assert materials.phonon.summary_rester.delta_catalog is cat + + +def test_get_delta_table_warns_on_label_mismatch(no_heartbeat): + rester = BaseRester(api_key="a" * 32) + lbl, _ = rester._get_delta_table("bucket", "some/prefix", label="first") + assert lbl == "first" + assert rester.delta_catalog.tables["first"].table_uri == "s3a://bucket/some/prefix/" + with pytest.warns(MPRestWarning, match="different label"): + lbl, _ = rester._get_delta_table("bucket", "some/prefix/", label="second") + assert lbl == "first" + + +def test_query_delta_single_retries_and_wraps_errors(no_heartbeat): + rester = BaseRester(api_key="a" * 32) + lbl, _ = rester._get_delta_table("bucket", "prefix", label="tbl") + + FakeQueryBuilder.failures = [DeltaError(REAL_ERROR)] + assert rester._query_delta_single("SELECT 1", label=lbl).num_rows == 1 + + FakeQueryBuilder.failures = [DeltaError(REAL_ERROR), DeltaError(REAL_ERROR)] + with pytest.raises(MPRestError, match="refreshed and the query retried"): + rester._query_delta_single("SELECT 1", label=lbl) + + FakeQueryBuilder.failures = [DeltaError("request timed out")] + with pytest.raises(MPRestError, match="increasing the 'timeout'"): + rester._query_delta_single("SELECT 1", label=lbl) diff --git a/tests/client/core/test_schemas.py b/tests/client/core/test_schemas.py index 574e5229..55018a71 100644 --- a/tests/client/core/test_schemas.py +++ b/tests/client/core/test_schemas.py @@ -7,22 +7,22 @@ from mp_api.client.core.schemas import _DictLikeAccess, _convert_to_model -class TestClass(_DictLikeAccess): +class DummyClass(_DictLikeAccess): a: int b: float c: list[str] def test_dict_like_access(): - instance = TestClass(a=1, b=2.0, c=["a", "b", "c"]) + instance = DummyClass(a=1, b=2.0, c=["a", "b", "c"]) assert isinstance(instance, BaseModel) assert all( getattr(instance, field_name) == instance[field_name] and instance[field_name] == instance.get(field_name) - for field_name in TestClass.model_fields + for field_name in DummyClass.model_fields ) - as_str = """TestClass( + as_str = """DummyClass( a (int) : 1 b (float) : 2.0 c (list) : ['a', 'b', 'c'] @@ -30,7 +30,9 @@ def test_dict_like_access(): assert str(instance) == as_str assert repr(instance) == as_str - with pytest.raises(AttributeError, match="'TestClass' object has no attribute 'd'"): + with pytest.raises( + AttributeError, match="'DummyClass' object has no attribute 'd'" + ): instance.d assert instance.get("d", None) == None @@ -53,7 +55,7 @@ def test_model_generation(): {k: v for k, v in doc.items() if k != "b"} for doc in get_data() ), }.items(): - as_models = _convert_to_model(trial_data, TestClass, model_name=test_type) + as_models = _convert_to_model(trial_data, DummyClass, model_name=test_type) assert all(isinstance(doc, BaseModel) for doc in as_models) assert all(doc.__class__.__name__ == test_type for doc in as_models) @@ -73,20 +75,20 @@ def test_model_generation(): else: assert all( getattr(doc, k) and doc.get(k) - for k in TestClass.model_fields + for k in DummyClass.model_fields for doc in as_models ) assert all( substr in str(doc) - for substr in ("Fields not requested", "TestClass", test_type) + for substr in ("Fields not requested", "DummyClass", test_type) for doc in as_models ) # Test requesting unavailable fields as_models = _convert_to_model( [{k: v for k, v in doc.items() if k != "b"} for doc in get_data()], - TestClass, + DummyClass, requested_fields=["b"], ) @@ -96,7 +98,7 @@ def test_model_generation(): # Test accessing fields that weren't requested as_models = _convert_to_model( [{k: v for k, v in doc.items() if k == "b"} for doc in get_data()], - TestClass, + DummyClass, requested_fields=["b"], ) with pytest.raises( @@ -105,4 +107,4 @@ def test_model_generation(): as_models[0].a # Ensure graceful handling of empty iterator input (no docs returned) - assert _convert_to_model(iter([]), TestClass) == [] + assert _convert_to_model(iter([]), DummyClass) == [] diff --git a/tests/client/materials/test_electrodes.py b/tests/client/materials/test_electrodes.py index cb37d486..6e91b7f2 100644 --- a/tests/client/materials/test_electrodes.py +++ b/tests/client/materials/test_electrodes.py @@ -1,4 +1,5 @@ import os +import warnings import pytest from pymatgen.core.periodic_table import Element @@ -74,19 +75,21 @@ def test_insertion_client(insertion_rester): @requires_api_key def test_conversion_client(conversion_rester): - search_method = conversion_rester.search - - excl = ConversionElectrodeRester._exclude_search_fields - client_search_testing( - search_method=search_method, - excluded_params=excluded_params + excl, - alt_name_dict=alt_name_dict, - custom_field_tests={ - "battery_ids": ["mp-1067_Al"], - "working_ion": Element("Li"), - }, - sub_doc_fields=sub_doc_fields, - ) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=UserWarning) + search_method = conversion_rester.search + + excl = ConversionElectrodeRester._exclude_search_fields + client_search_testing( + search_method=search_method, + excluded_params=excluded_params + excl, + alt_name_dict=alt_name_dict, + custom_field_tests={ + "battery_ids": ["mp-1067_Al"], + "working_ion": Element("Li"), + }, + sub_doc_fields=sub_doc_fields, + ) @requires_api_key diff --git a/tests/client/molecules/test_jcesr.py b/tests/client/molecules/test_jcesr.py index cd7dd97b..50866f7a 100644 --- a/tests/client/molecules/test_jcesr.py +++ b/tests/client/molecules/test_jcesr.py @@ -1,4 +1,5 @@ import os +import warnings import pytest from pymatgen.core.periodic_table import Element @@ -15,9 +16,11 @@ @pytest.fixture def rester(): - rester = JcesrMoleculesRester() - yield rester - rester.session.close() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + rester = JcesrMoleculesRester() + yield rester + rester.session.close() excluded_params = [ @@ -64,8 +67,10 @@ def test_warning(): @requires_api_key def test_pagination(): - with JcesrMoleculesRester() as rester: - client_pagination(rester.search, "task_id") + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with JcesrMoleculesRester() as rester: + client_pagination(rester.search, "task_id") @requires_api_key @@ -76,5 +81,7 @@ def test_pagination(): ], ) def test_sort(sort_field): - with JcesrMoleculesRester() as rester: - client_sort(rester.search, sort_field, default_fields=()) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with JcesrMoleculesRester() as rester: + client_sort(rester.search, sort_field, default_fields=()) diff --git a/tests/client/molecules/test_molecules.py b/tests/client/molecules/test_molecules.py index 5d5cd098..88643a8d 100644 --- a/tests/client/molecules/test_molecules.py +++ b/tests/client/molecules/test_molecules.py @@ -4,15 +4,18 @@ point for other resters, therefore we include only basic tests here. """ +import warnings from mp_api.client.routes.molecules.molecules import MoleculeRester from mp_api.client.routes.molecules import MOLECULES_RESTERS def test_molecule_rester(): - with MoleculeRester() as rester: - assert all(sub_rester in dir(rester) for sub_rester in MOLECULES_RESTERS) + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with MoleculeRester() as rester: + assert all(sub_rester in dir(rester) for sub_rester in MOLECULES_RESTERS) - assert all( - getattr(rester, k)._class_name == lazy_obj._class_name - for k, lazy_obj in MOLECULES_RESTERS.items() - ) + assert all( + getattr(rester, k)._class_name == lazy_obj._class_name + for k, lazy_obj in MOLECULES_RESTERS.items() + ) diff --git a/tests/client/test_mprester.py b/tests/client/test_mprester.py index 1b807cd6..3dd0b035 100644 --- a/tests/client/test_mprester.py +++ b/tests/client/test_mprester.py @@ -35,7 +35,9 @@ ConstantEnergyAdjustment, GibbsComputedStructureEntry, ) -from pymatgen.entries.mixing_scheme import MaterialsProjectDFTMixingScheme +from pymatgen.analysis.compatibility.mixing_scheme import ( + MaterialsProjectDFTMixingScheme, +) from pymatgen.io.cif import CifParser from pymatgen.io.vasp import Chgcar @@ -144,46 +146,53 @@ def test_find_structure( _atom_site_occupancy Ne Ne0 1 0.00000000 0.00000000 -0.00000000 1 """ - struct_from_cif = CifParser.from_str(cif_str).parse_structures(primitive=True)[ - 0 - ] - temp_file = NamedTemporaryFile(suffix=".cif") - with open(temp_file.name, "wt") as f: - f.write(cif_str) - f.seek(0) - - for struct_or_path, use_document_model in [ - (temp_file.name, True), - (struct_from_cif, False), - ]: - with MPRester(use_document_model=use_document_model) as mpr: - data = mpr.find_structure(struct_or_path) - assert isinstance(data, str) and data == "mp-111" - f.close() + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=UserWarning) + struct_from_cif = CifParser.from_str(cif_str).parse_structures( + primitive=True + )[0] + temp_file = NamedTemporaryFile(suffix=".cif") + with open(temp_file.name, "wt") as f: + f.write(cif_str) + f.seek(0) + + for struct_or_path, use_document_model in [ + (temp_file.name, True), + (struct_from_cif, False), + ]: + with MPRester(use_document_model=use_document_model) as mpr: + data = mpr.find_structure(struct_or_path) + assert isinstance(data, str) and data == "mp-111" + + f.close() - with pytest.raises(MPRestError, match="Provide filename or Structure object."): - mpr.find_structure(struct_from_cif.as_dict()) + with pytest.raises( + MPRestError, match="Provide filename or Structure object." + ): + mpr.find_structure(struct_from_cif.as_dict()) - with pytest.raises(MPRestError, match="`allow_multiple_results` must be a"): - mpr.find_structure(struct_from_cif, allow_multiple_results=1.0) + with pytest.raises(MPRestError, match="`allow_multiple_results` must be a"): + mpr.find_structure(struct_from_cif, allow_multiple_results=1.0) - assert ( - len( - mpr.find_structure( - struct_from_cif.copy().replace_species({"Ne": "K"}), - allow_multiple_results=2, + assert ( + len( + mpr.find_structure( + struct_from_cif.copy().replace_species({"Ne": "K"}), + allow_multiple_results=2, + ) ) + <= 2 ) - <= 2 - ) def test_get_bandstructure_by_material_id(self, mpr): - bs = mpr.get_bandstructure_by_material_id("mp-149") - assert isinstance(bs, BandStructureSymmLine) - bs_uniform = mpr.get_bandstructure_by_material_id("mp-149", line_mode=False) - assert isinstance(bs_uniform, BandStructure) - assert not isinstance(bs_uniform, BandStructureSymmLine) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=UserWarning) + bs = mpr.get_bandstructure_by_material_id("mp-149") + assert isinstance(bs, BandStructureSymmLine) + bs_uniform = mpr.get_bandstructure_by_material_id("mp-149", line_mode=False) + assert isinstance(bs_uniform, BandStructure) + assert not isinstance(bs_uniform, BandStructureSymmLine) def test_get_dos_by_id(self, mpr): dos = mpr.get_dos_by_material_id("mp-149") @@ -282,7 +291,9 @@ def test_get_entries_in_chemsys(self, mpr): MPRestWarning, match="The default thermo type when retrieving entries" ): entries = mpr.get_entries_in_chemsys(syms) - entries2 = mpr.get_entries_in_chemsys(syms2) + entries2 = mpr.get_entries_in_chemsys( + syms2, additional_criteria={"thermo_types": [ThermoType.GGA_GGA_U_R2SCAN]} + ) elements = {Element(sym) for sym in syms} for e in entries: assert isinstance(e, ComputedEntry) @@ -292,7 +303,11 @@ def test_get_entries_in_chemsys(self, mpr): e2 = {i.entry_id for i in entries2} assert e1 == e2 - gibbs_entries = mpr.get_entries_in_chemsys(syms2, use_gibbs=500) + gibbs_entries = mpr.get_entries_in_chemsys( + syms2, + use_gibbs=500, + additional_criteria={"thermo_types": [ThermoType.GGA_GGA_U_R2SCAN]}, + ) for e in gibbs_entries: assert isinstance(e, GibbsComputedStructureEntry) @@ -308,7 +323,10 @@ def test_get_entries_in_chemsys_mixed_hull(self, mpr): chemical system that entry's thermo doc was built for, so pooling the served entries across subsystems put Cs2TiI6 ~4.6 eV/atom above the hull instead of on it. """ - entries = mpr.get_entries_in_chemsys("Cs-Ti-I") + entries = mpr.get_entries_in_chemsys( + "Cs-Ti-I", + additional_criteria={"thermo_types": [ThermoType.GGA_GGA_U_R2SCAN]}, + ) phase_diagram = PhaseDiagram(entries) host = next(e for e in entries if e.composition.reduced_formula == "Cs2TiI6") assert phase_diagram.get_e_above_hull(host) == pytest.approx(0.0, abs=1e-6) @@ -334,7 +352,9 @@ def test_get_entries_in_chemsys_mixed_hull(self, mpr): all_fields=False, fields=["material_id", "energy_above_hull"], ) - entries = mpr.get_entries_in_chemsys("H-O") + entries = mpr.get_entries_in_chemsys( + "H-O", additional_criteria={"thermo_types": [ThermoType.GGA_GGA_U_R2SCAN]} + ) phase_diagram = PhaseDiagram(entries) by_mpid = defaultdict(list) for entry in entries: @@ -352,7 +372,11 @@ def test_get_entries_in_chemsys_mixed_hull(self, mpr): # uncorrected mixed entries cannot be placed on a common scale, so a warning is thrown: with pytest.warns(MPRestWarning, match="common energy scale"): - mpr.get_entries_in_chemsys("Cs-Ti-I", compatible_only=False) + mpr.get_entries_in_chemsys( + "Cs-Ti-I", + compatible_only=False, + additional_criteria={"thermo_types": [ThermoType.GGA_GGA_U_R2SCAN]}, + ) def test_get_entries_in_chemsys_decorated_served_pd(self, mpr): """ @@ -360,9 +384,14 @@ def test_get_entries_in_chemsys_decorated_served_pd(self, mpr): the pre-built phase diagram (and decorated post-hoc), rather than falling back to re-applying the mixing scheme locally (which can differ from MP's hull). """ - entries = mpr.get_entries_in_chemsys("H-O") + entries = mpr.get_entries_in_chemsys( + "H-O", additional_criteria={"thermo_types": [ThermoType.GGA_GGA_U_R2SCAN]} + ) decorated = mpr.get_entries_in_chemsys( - "H-O", property_data=["energy_above_hull"], conventional_unit_cell=True + "H-O", + property_data=["energy_above_hull"], + conventional_unit_cell=True, + additional_criteria={"thermo_types": [ThermoType.GGA_GGA_U_R2SCAN]}, ) served = { str(e.entry_id): ( @@ -662,8 +691,6 @@ def test_get_cohesive_energy(self): with MPRester() as mpr: mpr.get_cohesive_energy("mp-1") - # SOMETHING IS OFF HERE FOR THE MIXING SCHEME - # MIXING SCHEME TEST IS FLAKY, PASSES ROUGHLY 20% OF THE TIME @pytest.mark.parametrize( "thermo_type", ["GGA_GGA+U", ThermoType.GGA_GGA_U_R2SCAN, "r2SCAN"] ) @@ -673,83 +700,80 @@ def test_get_stability(self, thermo_type): to include more diverse chemical environments and thermo types which reflect the scope of the current MP database. """ - if ( - isinstance(thermo_type, ThermoType) - and thermo_type == ThermoType.GGA_GGA_U_R2SCAN - ): - pytest.skip("See comments about flakiness for mixing scheme") - with MPRester() as mpr: - - # No golden test data. Always test on fetched thermo data - chemsys_to_test: set[str] = { - doc.chemsys - for doc in mpr.materials.thermo.search( - thermo_types=[thermo_type], - num_elements=2, - num_chunks=1, - chunk_size=4, - fields=["chemsys"], - ) - } + with warnings.catch_warnings(): + # ignore some common pmg warnings: failed to guess oxi states, discarding entries, etc. + warnings.filterwarnings("ignore", category=UserWarning) - for chemsys in chemsys_to_test: - - # RETURN ORDER NOT DETERMINISTIC - entries = mpr.get_entries_in_chemsys( - chemsys, additional_criteria={"thermo_types": [thermo_type]} - ) - - modified_entries = [ - ComputedEntry( - entry.composition, - entry.uncorrected_energy + 0.01, - parameters=entry.parameters, - entry_id=f"mod_{entry.entry_id}", + with MPRester() as mpr: + # No golden test data. Always test on fetched thermo data + chemsys_to_test: set[str] = { + doc.chemsys + for doc in mpr.materials.thermo.search( + thermo_types=[thermo_type], + num_elements=2, + num_chunks=1, + chunk_size=4, + fields=["chemsys"], ) - for entry in entries - # MIXING SCHEME - ONLY PASSES IF A "GOOD" ENTRY IS RETURNED FIRST?? - if entry.entry_id == entries[0].entry_id - ] - - if ( - all(len(entry.composition.elements) == 1 for entry in entries) - and chemsys.count("-") > 0 - ): - # For a multi-element chemsys with no multinaries, only elementals, - # there should be no phase diagram data available. - with pytest.warns( - MPRestWarning, match="No phase diagram data available" - ): - mpr.get_stability(modified_entries, thermo_type=thermo_type) - return + } - else: - rester_ehulls = mpr.get_stability( - modified_entries, thermo_type=thermo_type + for chemsys in chemsys_to_test: + # RETURN ORDER NOT DETERMINISTIC + entries = mpr.get_entries_in_chemsys( + chemsys, additional_criteria={"thermo_types": [thermo_type]} ) - all_entries = entries + modified_entries - - compat = None - if thermo_type == "GGA_GGA+U": - compat = MaterialsProject2020Compatibility() - elif thermo_type == "GGA_GGA+U_R2SCAN": - compat = MaterialsProjectDFTMixingScheme(run_type_2="r2SCAN") - - if compat: - all_entries = compat.process_entries(all_entries) - - pd = PhaseDiagram(all_entries) - for entry in all_entries: - if str(entry.entry_id).startswith("mod"): - for dct in rester_ehulls: - if dct["entry_id"] == entry.entry_id: - data = dct - break - assert pd.get_e_above_hull(entry) == pytest.approx( - data["e_above_hull"] - ) + modified_entries = [ + ComputedEntry( + entry.composition, + entry.uncorrected_energy + 0.01, + parameters=entry.parameters, + entry_id=f"mod_{entry.entry_id}", + ) + for entry in entries + # MIXING SCHEME - ONLY PASSES IF A "GOOD" ENTRY IS RETURNED FIRST?? + if entry.entry_id == entries[0].entry_id + ] + + if ( + all(len(entry.composition.elements) == 1 for entry in entries) + and chemsys.count("-") > 0 + ): + # For a multi-element chemsys with no multinaries, only elementals, + # there should be no phase diagram data available. + with pytest.warns( + MPRestWarning, match="No phase diagram data available" + ): + mpr.get_stability(modified_entries, thermo_type=thermo_type) + return + + else: + rester_ehulls = mpr.get_stability( + modified_entries, thermo_type=thermo_type + ) + + all_entries = entries + modified_entries + + compat = None + if thermo_type == "GGA_GGA+U": + compat = MaterialsProject2020Compatibility() + elif thermo_type == "GGA_GGA+U_R2SCAN": + compat = MaterialsProjectDFTMixingScheme(run_type_2="r2SCAN") + + if compat: + all_entries = compat.process_entries(all_entries) + + pd = PhaseDiagram(all_entries) + for entry in all_entries: + if str(entry.entry_id).startswith("mod"): + for dct in rester_ehulls: + if dct["entry_id"] == entry.entry_id: + data = dct + break + assert pd.get_e_above_hull(entry) == pytest.approx( + data["e_above_hull"] + ) @pytest.mark.parametrize( "mpid, working_ion, thermo_type", @@ -861,7 +885,7 @@ def test_warnings_exceptions(self): with MPRester() as mpr: with pytest.raises( NotImplementedError, - match="The MPRester\(\).query method has been replaced", + match=r"The MPRester\(\).query method has been replaced", ): mpr.query(some_field=1.0) @@ -872,7 +896,7 @@ def test_warnings_exceptions(self): for attr in mpr._deprecated_attributes: with pytest.warns( - DeprecationWarning, match="Accessing.*data through MPRester\..*" + DeprecationWarning, match=r"Accessing.*data through MPRester\..*" ): getattr(mpr, attr, None)