From 8ada1463762577e786717d3827239f370ef8fa26 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 28 Jun 2026 04:01:18 +0200 Subject: [PATCH 1/2] feat: lazy package init to keep `import python_utils` light Defer all submodule/function imports via PEP 562 (`__getattr__`/`__dir__`) so `import python_utils` no longer eagerly imports every submodule, and move `asyncio` (and `python_utils.aio`) imports inside `python_utils.time`'s async helpers. Consumers that only need the synchronous utilities no longer pay the cost of importing `asyncio`. - __init__.py: PEP 562 lazy loading; `__version__` kept eager (cheap metadata). Public `__all__` is unchanged and `dir()` still lists the lazy `containers`/`exceptions` submodules so `import_global()` keeps working. - time.py: `aio_timeout_generator(iterable=...)` now defaults to `None` and resolves to `aio.acount` lazily (the default cannot reference `aio.acount` without importing asyncio at module load); `asyncio` is imported inside the two async helpers. - tests: regression tests proving a bare `import python_utils` and `import python_utils.time` pull in neither `asyncio` nor `typing_extensions`. Public API verified against the 4.0.0 base via an export/signature diff: the only differences are the two inherent consequences of deferring asyncio -- `aio_timeout_generator`'s default iterable is `None` instead of `aio.acount`, and `python_utils.time` no longer re-exposes the `aio`/`asyncio` modules. --- _python_utils_tests/test_lazy_imports.py | 96 ++++++++++++++ python_utils/__init__.py | 160 ++++++++++++++++++----- python_utils/time.py | 23 +++- 3 files changed, 243 insertions(+), 36 deletions(-) create mode 100644 _python_utils_tests/test_lazy_imports.py diff --git a/_python_utils_tests/test_lazy_imports.py b/_python_utils_tests/test_lazy_imports.py new file mode 100644 index 0000000..f5e5c5a --- /dev/null +++ b/_python_utils_tests/test_lazy_imports.py @@ -0,0 +1,96 @@ +"""Tests for the lazy-import machinery that keeps `import python_utils` light +(PEP 562 `__getattr__`/`__dir__` in the package, deferred ``asyncio`` in +``python_utils.time``). +""" + +import collections.abc +import os +import subprocess +import sys + +import pytest + +import python_utils + + +def _run_clean(code: str) -> subprocess.CompletedProcess[str]: + # Run in a fresh interpreter: the test session itself has long since + # imported asyncio/typing_extensions, so in-process checks are useless. + env = {**os.environ, 'PYTHONPATH': os.pathsep.join(sys.path)} + return subprocess.run( + [sys.executable, '-c', code], + capture_output=True, + text=True, + env=env, + ) + + +def test_package_lazy_attribute_access() -> None: + # Submodule access and exported-name access both resolve via __getattr__. + aio = python_utils.aio + assert python_utils.aio is aio # repeated access returns the cached module + assert callable(python_utils.acount) + assert isinstance(python_utils.__version__, str) + missing = 'definitely_not_a_real_attribute' + with pytest.raises(AttributeError): + getattr(python_utils, missing) + + +def test_bare_import_stays_light() -> None: + # Importing the package must not eagerly pull in heavy/optional deps. + result = _run_clean( + 'import sys, python_utils\n' + "assert 'asyncio' not in sys.modules, " + "sorted(m for m in sys.modules if m.startswith('asyncio'))\n" + "assert 'typing_extensions' not in sys.modules\n" + ) + assert result.returncode == 0, result.stderr + + +def test_importing_time_submodule_avoids_asyncio() -> None: + # Importing python_utils.time for its synchronous helpers must not import + # asyncio; the async helpers import it lazily inside their own bodies. + result = _run_clean( + 'import sys, python_utils.time\n' + "assert 'asyncio' not in sys.modules, " + "sorted(m for m in sys.modules if m.startswith('asyncio'))\n" + ) + assert result.returncode == 0, result.stderr + + +def test_first_access_caches_into_module_dict() -> None: + # PEP 562 __getattr__ runs once: the resolved object is cached in the + # module namespace so subsequent lookups skip __getattr__ entirely. + module = python_utils.time + assert python_utils.__dict__['time'] is module + + func = python_utils.format_time + assert python_utils.__dict__['format_time'] is func + + +def test_dir_lists_lazy_submodules() -> None: + # Lazy submodules that are not in __all__ (e.g. ``containers`` and + # ``exceptions``) must still be discoverable via ``dir``; tools such as + # ``import_global`` intersect requested names with ``dir(module)``. + names = set(dir(python_utils)) + assert {'containers', 'exceptions'} <= names + assert set(python_utils.__all__) <= names + + +@pytest.mark.asyncio +async def test_aio_timeout_generator_default_iterable() -> None: + # With no iterable the generator defaults to ``aio.acount`` -- exercising + # the lazy ``aio``/``asyncio`` import and the None-resolution branch. + count = 0 + generator: collections.abc.AsyncGenerator[object, None] = ( + python_utils.aio_timeout_generator(timeout=0.05, interval=0.0) + ) + async for _ in generator: + count += 1 + if count >= 2: + break + + assert count == 2 + + # Sanity: the re-exported type alias still resolves through the package. + assert python_utils.types.AsyncGenerator is not None diff --git a/python_utils/__init__.py b/python_utils/__init__.py index 3c6242d..a238ebf 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -2,6 +2,12 @@ This module initializes the `python_utils` package by importing various submodules and functions. +Imports are performed lazily (PEP 562): nothing is imported when you ``import +python_utils``; each submodule/function is loaded on first access. This keeps +``import python_utils`` cheap and, in particular, avoids eagerly importing +``asyncio`` (via the async helpers) for consumers that only need the +synchronous utilities. + Submodules:: aio @@ -52,40 +58,132 @@ LoggerBase """ -from . import ( - aio, - converters, - decorators, - formatters, - generators, - import_, - logger, - terminal, - time, - types, -) +import importlib as _importlib +import typing as _typing + from .__about__ import __version__ -from .aio import acount -from .containers import CastedDict, LazyCastedDict, UniqueList -from .converters import remap, scale_1024, to_float, to_int, to_str, to_unicode -from .decorators import listify, set_attributes -from .exceptions import raise_exception, reraise -from .formatters import camel_to_underscore, timesince -from .generators import abatcher, batcher -from .import_ import import_global -from .logger import Logged, LoggerBase -from .terminal import get_terminal_size -from .time import ( - aio_generator_timeout_detector, - aio_generator_timeout_detector_decorator, - aio_timeout_generator, - delta_to_seconds, - delta_to_seconds_or_none, - format_time, - timedelta_to_seconds, - timeout_generator, + +if _typing.TYPE_CHECKING: # pragma: no cover + # Eager imports for type checkers only; the runtime equivalents are loaded + # lazily by ``__getattr__`` below. Names appear in ``__all__`` so they are + # treated as re-exports (not unused imports). + from . import ( + aio, + converters, + decorators, + formatters, + generators, + import_, + logger, + terminal, + time, + types, + ) + from .aio import acount + from .containers import CastedDict, LazyCastedDict, UniqueList + from .converters import ( + remap, + scale_1024, + to_float, + to_int, + to_str, + to_unicode, + ) + from .decorators import listify, set_attributes + from .exceptions import raise_exception, reraise + from .formatters import camel_to_underscore, timesince + from .generators import abatcher, batcher + from .import_ import import_global + from .logger import Logged, LoggerBase + from .terminal import get_terminal_size + from .time import ( + aio_generator_timeout_detector, + aio_generator_timeout_detector_decorator, + aio_timeout_generator, + delta_to_seconds, + delta_to_seconds_or_none, + format_time, + timedelta_to_seconds, + timeout_generator, + ) + +#: Submodules that can be accessed as ``python_utils.``. +_SUBMODULES: frozenset[str] = frozenset( + { + 'aio', + 'containers', + 'converters', + 'decorators', + 'exceptions', + 'formatters', + 'generators', + 'import_', + 'logger', + 'terminal', + 'time', + 'types', + } ) +#: Exported name -> submodule it lives in. +_NAME_TO_MODULE: dict[str, str] = { + 'acount': 'aio', + 'CastedDict': 'containers', + 'LazyCastedDict': 'containers', + 'UniqueList': 'containers', + 'remap': 'converters', + 'scale_1024': 'converters', + 'to_float': 'converters', + 'to_int': 'converters', + 'to_str': 'converters', + 'to_unicode': 'converters', + 'listify': 'decorators', + 'set_attributes': 'decorators', + 'raise_exception': 'exceptions', + 'reraise': 'exceptions', + 'camel_to_underscore': 'formatters', + 'timesince': 'formatters', + 'abatcher': 'generators', + 'batcher': 'generators', + 'import_global': 'import_', + 'Logged': 'logger', + 'LoggerBase': 'logger', + 'get_terminal_size': 'terminal', + 'aio_generator_timeout_detector': 'time', + 'aio_generator_timeout_detector_decorator': 'time', + 'aio_timeout_generator': 'time', + 'delta_to_seconds': 'time', + 'delta_to_seconds_or_none': 'time', + 'format_time': 'time', + 'timedelta_to_seconds': 'time', + 'timeout_generator': 'time', +} + + +def __getattr__(name: str) -> _typing.Any: + """Lazily import submodules and their exported names on first access.""" + if name in _SUBMODULES: + module = _importlib.import_module(f'.{name}', __name__) + elif name in _NAME_TO_MODULE: + module = _importlib.import_module( + f'.{_NAME_TO_MODULE[name]}', __name__ + ) + value = getattr(module, name) + globals()[name] = value # cache so __getattr__ runs only once + return value + else: + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') + + globals()[name] = module + return module + + +def __dir__() -> list[str]: + return sorted( + set(globals()) | set(__all__) | _SUBMODULES | set(_NAME_TO_MODULE) + ) + + __all__ = [ 'CastedDict', 'LazyCastedDict', diff --git a/python_utils/time.py b/python_utils/time.py index 90f5ccd..5a2243b 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -16,7 +16,6 @@ """ # pyright: reportUnnecessaryIsInstance=false -import asyncio import collections.abc import datetime import functools @@ -25,7 +24,7 @@ import typing import python_utils -from python_utils import aio, exceptions, types +from python_utils import exceptions, types _T = typing.TypeVar('_T') _P = typing.ParamSpec('_P') @@ -256,9 +255,8 @@ async def aio_timeout_generator( timeout: types.delta_type, # noqa: ASYNC109 interval: types.delta_type = datetime.timedelta(seconds=1), iterable: collections.abc.AsyncIterable[_T] - | collections.abc.Callable[ - ..., collections.abc.AsyncIterable[_T] - ] = aio.acount, + | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]] + | None = None, interval_multiplier: float = 1.0, maximum_interval: types.delta_type | None = None, ) -> collections.abc.AsyncGenerator[_T, None]: @@ -276,6 +274,18 @@ async def aio_timeout_generator( effectively the same as the `timeout_generator` but it uses `async for` instead. """ + # Imported lazily so that importing `python_utils.time` for its + # synchronous helpers (e.g. ``format_time``) does not pull in ``asyncio``. + import asyncio + + from python_utils import aio + + if iterable is None: + iterable = typing.cast( + collections.abc.Callable[[], collections.abc.AsyncIterable[_T]], + aio.acount, + ) + float_interval: float = delta_to_seconds(interval) float_maximum_interval: float | None = delta_to_seconds_or_none( maximum_interval @@ -323,6 +333,9 @@ async def aio_generator_timeout_detector( If `on_timeout` is `None`, the exception is silently ignored and the generator will finish as normal. """ + # Imported lazily so importing `python_utils.time` stays asyncio-free. + import asyncio + if total_timeout is None: total_timeout_end = None else: From 047d64933386edf9997b42c2ae4fd3a0c3ee5775 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 29 Jun 2026 14:31:19 +0200 Subject: [PATCH 2/2] perf: defer typing_extensions and lazy __version__ to keep imports light Builds on the lazy package init: removes typing_extensions from the submodule import path and defers __version__. - Add stdlib-only python_utils/_aliases.py holding the lightweight type aliases; python_utils.types re-exports them (explicit `X as X` idiom) and keeps its eager typing_extensions facade + runtime overrides unchanged. - Rewire time/converters/formatters/generators/import_ off python_utils.types onto _aliases, so importing them no longer pulls typing_extensions. - Move logger's only typing_extensions use (Logged.__new__ -> Self) into TYPE_CHECKING (Self is a new addition; runtime introspection of __new__'s return type is not a compat obligation). - Make python_utils.__version__ lazy via the PEP 562 __getattr__ so bare `import python_utils` no longer calls importlib.metadata.version() (~89 -> ~8 modules added to sys.modules). - Add a deterministic import-footprint regression gate + get_type_hints smoke. Public API verified byte-identical to develop via an export/signature manifest diff, except: (a) aio_timeout_generator's default iterable (from the lazy asyncio change) and (b) converters' raw __annotations__ strings now name the bare aliases (get_type_hints resolves identically). `from __future__ import annotations` is limited to where required (logger, containers) so inspect.signature keeps returning evaluated type objects elsewhere. The undocumented python_utils..types re-export attribute is removed. --- _python_utils_tests/test_aliases.py | 57 +++++++++++ _python_utils_tests/test_import_footprint.py | 100 +++++++++++++++++++ python_utils/__init__.py | 4 +- python_utils/_aliases.py | 43 ++++++++ python_utils/converters.py | 20 ++-- python_utils/formatters.py | 6 +- python_utils/generators.py | 16 +-- python_utils/import_.py | 8 +- python_utils/logger.py | 5 +- python_utils/time.py | 40 ++++---- python_utils/types.py | 34 +++---- 11 files changed, 266 insertions(+), 67 deletions(-) create mode 100644 _python_utils_tests/test_aliases.py create mode 100644 _python_utils_tests/test_import_footprint.py create mode 100644 python_utils/_aliases.py diff --git a/_python_utils_tests/test_aliases.py b/_python_utils_tests/test_aliases.py new file mode 100644 index 0000000..301f9bd --- /dev/null +++ b/_python_utils_tests/test_aliases.py @@ -0,0 +1,57 @@ +"""The lightweight alias module must define the public type aliases without +pulling in typing_extensions, so importers stay light. +""" + +import os +import subprocess +import sys + + +def test_aliases_do_not_import_typing_extensions() -> None: + code = ( + 'import sys, python_utils._aliases\n' + "assert 'typing_extensions' not in sys.modules\n" + ) + result = subprocess.run( + [sys.executable, '-c', code], + capture_output=True, + text=True, + env={**os.environ, 'PYTHONPATH': os.pathsep.join(sys.path)}, + ) + assert result.returncode == 0, result.stderr + + +def test_aliases_values() -> None: + from python_utils import _aliases + + assert _aliases.Number == (int | float) + assert _aliases.delta_type == ( + __import__('datetime').timedelta | int | float + ) + assert set(_aliases.__all__) == { + 'Scope', + 'OptionalScope', + 'Number', + 'DecimalNumber', + 'ExceptionType', + 'ExceptionsType', + 'StringTypes', + 'delta_type', + 'timestamp_type', + } + + +def test_types_reexports_aliases_identically() -> None: + from python_utils import _aliases, types + + for name in _aliases.__all__: + assert getattr(types, name) is getattr(_aliases, name), name + + +def test_types_still_exposes_typing_extensions_surface() -> None: + # The facade must keep re-exporting typing_extensions (e.g. Self). + # ``hasattr`` (not ``types.Self``) avoids basedpyright's + # reportUnknownMemberType, since the wildcard re-export has no static type. + from python_utils import types + + assert hasattr(types, 'Self') diff --git a/_python_utils_tests/test_import_footprint.py b/_python_utils_tests/test_import_footprint.py new file mode 100644 index 0000000..bc55d60 --- /dev/null +++ b/_python_utils_tests/test_import_footprint.py @@ -0,0 +1,100 @@ +"""Deterministic import-footprint regression gate. + +Each target module is imported in a clean subprocess; a per-target denylist of +heavy modules must be absent from sys.modules afterward. This is the CI +performance guard: it fails the moment an eager heavy import is reintroduced. +""" + +import json +import os +import subprocess +import sys + +import pytest + +# (import target, modules that must be ABSENT from sys.modules afterward) +FOOTPRINT_CASES = [ + ('python_utils', ('typing_extensions', 'asyncio')), + ('python_utils.time', ('typing_extensions', 'asyncio')), + ('python_utils.logger', ('typing_extensions',)), + ('python_utils.converters', ('typing_extensions', 'asyncio')), + ('python_utils.formatters', ('typing_extensions', 'asyncio')), + ('python_utils.import_', ('typing_extensions', 'asyncio')), + ('python_utils.terminal', ('typing_extensions',)), + ('python_utils.containers', ('typing_extensions', 'asyncio')), + ('python_utils.decorators', ('typing_extensions', 'asyncio')), + ('python_utils.exceptions', ('typing_extensions', 'asyncio')), + # aio, generators legitimately use asyncio; only typing_extensions denied. + ('python_utils.aio', ('typing_extensions',)), + ('python_utils.generators', ('typing_extensions',)), +] + + +def _modules_after_import(target: str) -> set[str]: + code = ( + f'import sys, {target}\n' + 'import json\n' + 'print(json.dumps(sorted(sys.modules)))\n' + ) + result = subprocess.run( + [sys.executable, '-c', code], + capture_output=True, + text=True, + env={**os.environ, 'PYTHONPATH': os.pathsep.join(sys.path)}, + ) + assert result.returncode == 0, result.stderr + + return set(json.loads(result.stdout)) + + +@pytest.mark.parametrize(('target', 'denied'), FOOTPRINT_CASES) +def test_import_footprint(target: str, denied: tuple[str, ...]) -> None: + present = _modules_after_import(target) + leaked = [m for m in denied if m in present] + assert not leaked, f'{target} eagerly imported {leaked}' + + +def test_bare_import_module_count_under_budget() -> None: + # Coarse bloat tripwire (denylist above is the real guard). Cap tightened + # now that __version__ is lazy (importlib.metadata no longer pulled on bare + # import). Bump only if a new Python version legitimately adds startup + # modules. Measures modules ADDED by importing python_utils. + added = len(_modules_after_import('python_utils')) - len( + _modules_after_import('sys') + ) + assert added < 40, f'python_utils added {added} modules to sys.modules' + + +def test_bare_import_does_not_pull_importlib_metadata() -> None: + # __version__ is resolved lazily; bare import must not call + # importlib.metadata.version() (which drags in email/zipfile/json/...). + present = _modules_after_import('python_utils') + assert 'importlib.metadata' not in present + + +def test_version_resolves_correctly() -> None: + import python_utils + + assert isinstance(python_utils.__version__, str) + assert python_utils.__version__ # non-empty + + +PUBLIC_CALLABLES_TO_INTROSPECT = [ + ('python_utils.time', 'timeout_generator'), + ('python_utils.time', 'aio_timeout_generator'), + ('python_utils.time', 'format_time'), + ('python_utils.converters', 'remap'), + ('python_utils.converters', 'to_int'), + ('python_utils.formatters', 'timesince'), + ('python_utils.import_', 'import_global'), +] + + +@pytest.mark.parametrize(('module', 'name'), PUBLIC_CALLABLES_TO_INTROSPECT) +def test_get_type_hints_still_resolves(module: str, name: str) -> None: + import importlib + import typing + + obj = getattr(importlib.import_module(module), name) + # Must not raise NameError now that type imports moved/changed. + typing.get_type_hints(obj) diff --git a/python_utils/__init__.py b/python_utils/__init__.py index a238ebf..300f54a 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -61,8 +61,6 @@ import importlib as _importlib import typing as _typing -from .__about__ import __version__ - if _typing.TYPE_CHECKING: # pragma: no cover # Eager imports for type checkers only; the runtime equivalents are loaded # lazily by ``__getattr__`` below. Names appear in ``__all__`` so they are @@ -79,6 +77,7 @@ time, types, ) + from .__about__ import __version__ from .aio import acount from .containers import CastedDict, LazyCastedDict, UniqueList from .converters import ( @@ -127,6 +126,7 @@ #: Exported name -> submodule it lives in. _NAME_TO_MODULE: dict[str, str] = { + '__version__': '__about__', 'acount': 'aio', 'CastedDict': 'containers', 'LazyCastedDict': 'containers', diff --git a/python_utils/_aliases.py b/python_utils/_aliases.py new file mode 100644 index 0000000..c0940fc --- /dev/null +++ b/python_utils/_aliases.py @@ -0,0 +1,43 @@ +"""Lightweight, stdlib-only type aliases shared across python_utils. + +These live here (rather than in ``python_utils.types``) so internal modules can +import them without dragging in ``typing_extensions``. ``python_utils.types`` +re-exports everything defined here, so the public names are unchanged. +""" + +from __future__ import annotations + +import datetime +import decimal +from typing import Any + +__all__ = [ + 'DecimalNumber', + 'ExceptionType', + 'ExceptionsType', + 'Number', + 'OptionalScope', + 'Scope', + 'StringTypes', + 'delta_type', + 'timestamp_type', +] + +Scope = dict[str, Any] +OptionalScope = Scope | None +Number = int | float +DecimalNumber = Number | decimal.Decimal +ExceptionType = type[Exception] +ExceptionsType = tuple[ExceptionType, ...] | ExceptionType +StringTypes = str | bytes + +delta_type = datetime.timedelta | int | float +timestamp_type = ( + datetime.timedelta + | datetime.date + | datetime.datetime + | str + | int + | float + | None +) diff --git a/python_utils/converters.py b/python_utils/converters.py index 4a65668..37dca0c 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -20,9 +20,9 @@ import re import typing -from . import types +from python_utils import _aliases -_TN = typing.TypeVar('_TN', bound=types.DecimalNumber) +_TN = typing.TypeVar('_TN', bound=_aliases.DecimalNumber) _RegexpType: typing.TypeAlias = ( re.Pattern[str] | str | typing.Literal[True] | None @@ -32,7 +32,7 @@ def to_int( input_: str | None = None, default: int = 0, - exception: types.ExceptionsType = (ValueError, TypeError), + exception: _aliases.ExceptionsType = (ValueError, TypeError), regexp: _RegexpType = None, ) -> int: r""" @@ -120,9 +120,9 @@ def to_int( def to_float( input_: str, default: int = 0, - exception: types.ExceptionsType = (ValueError, TypeError), + exception: _aliases.ExceptionsType = (ValueError, TypeError), regexp: _RegexpType = None, -) -> types.Number: +) -> _aliases.Number: r""" Convert the given `input_` to an integer or return default. @@ -192,7 +192,7 @@ def to_float( def to_unicode( - input_: types.StringTypes, + input_: _aliases.StringTypes, encoding: str = 'utf-8', errors: str = 'replace', ) -> str: @@ -222,7 +222,7 @@ def to_unicode( def to_str( - input_: types.StringTypes, + input_: _aliases.StringTypes, encoding: str = 'utf-8', errors: str = 'replace', ) -> bytes: @@ -252,9 +252,9 @@ def to_str( def scale_1024( - x: types.Number, + x: _aliases.Number, n_prefixes: int, -) -> tuple[types.Number, types.Number]: +) -> tuple[_aliases.Number, _aliases.Number]: """Scale a number down to a suitable size, based on powers of 1024. Returns the scaled number and the power of 1024 used. @@ -414,7 +414,7 @@ def remap( # pyright: ignore[reportInconsistentOverload] passed parameters are a `float`, otherwise the returned type will be `int`. """ - type_: type[types.DecimalNumber] + type_: type[_aliases.DecimalNumber] if ( isinstance(value, decimal.Decimal) or isinstance(old_min, decimal.Decimal) diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 3c1c2f3..667b376 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -13,7 +13,7 @@ import datetime import typing -from python_utils import types +from python_utils import _aliases def camel_to_underscore(name: str) -> str: @@ -56,9 +56,9 @@ def camel_to_underscore(name: str) -> str: def apply_recursive( function: collections.abc.Callable[[str], str], - data: types.OptionalScope = None, + data: _aliases.OptionalScope = None, **kwargs: typing.Any, -) -> types.OptionalScope: +) -> _aliases.OptionalScope: """ Apply a function to all keys in a scope recursively. diff --git a/python_utils/generators.py b/python_utils/generators.py index 146feca..eadc22c 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -17,7 +17,7 @@ import typing import python_utils -from python_utils import types +from python_utils import _aliases _T = typing.TypeVar('_T') @@ -26,7 +26,7 @@ async def abatcher( generator: collections.abc.AsyncGenerator[_T, None] | collections.abc.AsyncIterator[_T], batch_size: int | None = None, - interval: types.delta_type | None = None, + interval: _aliases.delta_type | None = None, ) -> collections.abc.AsyncGenerator[list[_T], None]: """ Asyncio generator wrapper that returns items with a given batch size or @@ -34,13 +34,13 @@ async def abatcher( Args: generator: The async generator or iterator to batch. - batch_size (types.Optional[int], optional): The number of items per + batch_size (typing.Optional[int], optional): The number of items per batch. Defaults to None. - interval (types.Optional[types.delta_type], optional): The time + interval (typing.Optional[_aliases.delta_type], optional): The time interval to wait before yielding a batch. Defaults to None. Yields: - types.AsyncGenerator[types.List[_T], None]: A generator that yields + collections.abc.AsyncGenerator[list[_T], None]: A generator that yields batches of items. """ batch: list[_T] = [] @@ -106,13 +106,13 @@ def batcher( Generator wrapper that returns items with a given batch size. Args: - iterable (types.Iterable[_T]): The iterable to batch. + iterable (collections.abc.Iterable[_T]): The iterable to batch. batch_size (int, optional): The number of items per batch. Defaults to 10. Yields: - types.Generator[types.List[_T], None, None]: A generator that yields - batches of items. + collections.abc.Generator[list[_T], None, None]: A generator that + yields batches of items. """ batch: list[_T] = [] for item in iterable: diff --git a/python_utils/import_.py b/python_utils/import_.py index 6f0f0a9..e1d0dc6 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -14,7 +14,7 @@ import typing -from . import types +from python_utils import _aliases class DummyError(Exception): @@ -28,9 +28,9 @@ class DummyError(Exception): def import_global( # noqa: C901 name: str, modules: list[str] | None = None, - exceptions: types.ExceptionsType = DummyError, - locals_: types.OptionalScope = None, - globals_: types.OptionalScope = None, + exceptions: _aliases.ExceptionsType = DummyError, + locals_: _aliases.OptionalScope = None, + globals_: _aliases.OptionalScope = None, level: int = -1, ) -> typing.Any: # sourcery skip: hoist-if-from-if """Import the requested items into the global scope. diff --git a/python_utils/logger.py b/python_utils/logger.py index 55b5c85..0c13bf0 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -25,13 +25,16 @@ >>> my_class.log(0, 'log') """ +from __future__ import annotations + import abc import collections.abc import logging import types import typing -import typing_extensions +if typing.TYPE_CHECKING: + import typing_extensions from . import decorators diff --git a/python_utils/time.py b/python_utils/time.py index 5a2243b..3df35c5 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -24,7 +24,7 @@ import typing import python_utils -from python_utils import exceptions, types +from python_utils import _aliases, exceptions _T = typing.TypeVar('_T') _P = typing.ParamSpec('_P') @@ -35,7 +35,7 @@ epoch = datetime.datetime(year=1970, month=1, day=1) -def timedelta_to_seconds(delta: datetime.timedelta) -> types.Number: +def timedelta_to_seconds(delta: datetime.timedelta) -> _aliases.Number: """Convert a timedelta to seconds with the microseconds as fraction. Note that this method has become largely obsolete with the @@ -61,7 +61,7 @@ def timedelta_to_seconds(delta: datetime.timedelta) -> types.Number: return total -def delta_to_seconds(interval: types.delta_type) -> types.Number: +def delta_to_seconds(interval: _aliases.delta_type) -> _aliases.Number: """ Convert a timedelta to seconds. @@ -85,8 +85,8 @@ def delta_to_seconds(interval: types.delta_type) -> types.Number: def delta_to_seconds_or_none( - interval: types.delta_type | None, -) -> types.Number | None: + interval: _aliases.delta_type | None, +) -> _aliases.Number | None: """Convert a timedelta to seconds or return None.""" if interval is None: return None @@ -95,7 +95,7 @@ def delta_to_seconds_or_none( def format_time( - timestamp: types.timestamp_type, + timestamp: _aliases.timestamp_type, precision: datetime.timedelta = datetime.timedelta(seconds=1), ) -> str: """Formats timedelta/datetime/seconds. @@ -187,14 +187,14 @@ def _to_iterable( def timeout_generator( - timeout: types.delta_type, - interval: types.delta_type = datetime.timedelta(seconds=1), + timeout: _aliases.delta_type, + interval: _aliases.delta_type = datetime.timedelta(seconds=1), iterable: collections.abc.Iterable[_T] | collections.abc.Callable[ [], collections.abc.Iterable[_T] ] = itertools.count, # type: ignore[assignment] interval_multiplier: float = 1.0, - maximum_interval: types.delta_type | None = None, + maximum_interval: _aliases.delta_type | None = None, ) -> collections.abc.Iterable[_T]: """ Generator that walks through the given iterable (a counter by default) @@ -252,13 +252,13 @@ def timeout_generator( async def aio_timeout_generator( - timeout: types.delta_type, # noqa: ASYNC109 - interval: types.delta_type = datetime.timedelta(seconds=1), + timeout: _aliases.delta_type, # noqa: ASYNC109 + interval: _aliases.delta_type = datetime.timedelta(seconds=1), iterable: collections.abc.AsyncIterable[_T] | collections.abc.Callable[..., collections.abc.AsyncIterable[_T]] | None = None, interval_multiplier: float = 1.0, - maximum_interval: types.delta_type | None = None, + maximum_interval: _aliases.delta_type | None = None, ) -> collections.abc.AsyncGenerator[_T, None]: """ Async generator that walks through the given async iterable (a counter by @@ -308,13 +308,13 @@ async def aio_timeout_generator( async def aio_generator_timeout_detector( generator: collections.abc.AsyncGenerator[_T, None], - timeout: types.delta_type | None = None, # noqa: ASYNC109 - total_timeout: types.delta_type | None = None, + timeout: _aliases.delta_type | None = None, # noqa: ASYNC109 + total_timeout: _aliases.delta_type | None = None, on_timeout: collections.abc.Callable[ [ collections.abc.AsyncGenerator[_T, None], - types.delta_type | None, - types.delta_type | None, + _aliases.delta_type | None, + _aliases.delta_type | None, BaseException, ], typing.Any, @@ -373,13 +373,13 @@ async def aio_generator_timeout_detector( def aio_generator_timeout_detector_decorator( - timeout: types.delta_type | None = None, - total_timeout: types.delta_type | None = None, + timeout: _aliases.delta_type | None = None, + total_timeout: _aliases.delta_type | None = None, on_timeout: collections.abc.Callable[ [ collections.abc.AsyncGenerator[typing.Any, None], - types.delta_type | None, - types.delta_type | None, + _aliases.delta_type | None, + _aliases.delta_type | None, BaseException, ], typing.Any, diff --git a/python_utils/types.py b/python_utils/types.py index a3c7882..1287316 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -12,8 +12,6 @@ # pyright: reportWildcardImportFromLibrary=false # ruff: noqa: F405 -import datetime -import decimal from re import Match, Pattern from types import * # pragma: no cover # noqa: F403 from typing import * # pragma: no cover # noqa: F403 @@ -30,23 +28,21 @@ from typing_extensions import * # type: ignore[no-redef,assignment] # noqa: F403 -Scope = dict[str, Any] -OptionalScope = Scope | None -Number = int | float -DecimalNumber = Number | decimal.Decimal -ExceptionType = type[Exception] -ExceptionsType = tuple[ExceptionType, ...] | ExceptionType -StringTypes = str | bytes - -delta_type = datetime.timedelta | int | float -timestamp_type = ( - datetime.timedelta - | datetime.date - | datetime.datetime - | str - | int - | float - | None +# Lightweight aliases live in a stdlib-only module so importers don't pull in +# typing_extensions; re-exported here to keep the public surface unchanged. +# The redundant `X as X` form marks these as intentional re-exports so strict +# type checkers and ruff see `python_utils.types.X` without adding them to +# `__all__` (which would change `from python_utils.types import *`). +from ._aliases import ( + DecimalNumber as DecimalNumber, + ExceptionType as ExceptionType, + ExceptionsType as ExceptionsType, + Number as Number, + OptionalScope as OptionalScope, + Scope as Scope, + StringTypes as StringTypes, + delta_type as delta_type, + timestamp_type as timestamp_type, ) __all__ = [