Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions pyathena/aio/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from botocore.exceptions import BotoCoreError, ClientError

from pyathena.aio.util import async_retry_api_call
from pyathena.common import BaseCursor
from pyathena.common import BaseCursor, CursorIterator
from pyathena.error import DatabaseError, OperationalError, ProgrammingError
from pyathena.glue import GlueMetadataClient
from pyathena.model import AthenaDatabase, AthenaQueryExecution, AthenaTableMetadata
Expand Down Expand Up @@ -617,13 +617,13 @@ async def athena_request(
)


class WithAsyncFetch(AioBaseCursor, WithResultSet):
class WithAsyncFetch(WithResultSet, AioBaseCursor, CursorIterator):

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Self-review round two (claims, callers, docs): FINDINGS, repaired

Base 809b38070ff7ed69b960f9f4318a82e03723fd2c, head 7d9ba8f55b82eae282567d61aae918ff797a595b → repaired head 85f0baa6b093adfc3b262d0f9a29dbdcd71ccb78 (commit message only; tree unchanged).

Claims checked:

  • PR table (v3.37.0 / pre-Share the pure cursor-base logic between the sync and asyncio cursors #883 / master structures): git show v3.37.0: and 51532ed7: of result_set.py and aio/common.py match the table.
  • "Members WithFetch and WithAsyncFetch duplicated before Share the pure cursor-base logic between the sync and asyncio cursors #883": 51532ed7 defines __init__, arraysize, result_set, query_id, rownumber, rowcount, close in both; all are now in WithResultSet only.
  • "No behavior change": member-resolution and instance-state comparison against master (round one).
  • "Share the pure cursor-base logic between the sync and asyncio cursors #883 release-note items no longer apply": a v3.37.0-style composition class MyCursor(BaseCursor, CursorIterator, WithResultSet) raises the MRO TypeError on master and works on this branch and on v3.37.0; a WithFetch subclass works on this branch and v3.37.0 (its arraysize=7 result differs only by SQL cursors ignore the arraysize keyword argument #897, already release-noted). Added this evidence to the PR body.
  • Docstrings: precedence note, **kwargs / DEFAULT_FETCH_SIZE in WithResultSet.__init__, "sync iteration" in WithFetch (from CursorIterator), "subclasses implement the fetch methods as coroutines" (all 5 aio cursors define async fetchone / fetchmany / fetchall): true.
  • Docs: no prose mentions WithFetch / WithResultSet beyond the autoclass entries; docs/api/connection.rst now renders fewer inherited members for the mixin. just docs build not run (docs sources unchanged).

Findings and repairs:

  1. Commit message said Share the pure cursor-base logic between the sync and asyncio cursors #883 "made the mixin a base class of BaseCursor and CursorIterator"; it made it a subclass. Amended (85f0baa), pushed with --force-with-lease.
  2. PR body said the base order is "the only structural difference from v3.37.0", ignoring the members moved into WithResultSet. Narrowed.

"""Base class of the asyncio SQL cursors.

Overrides ``executemany`` and ``cancel`` of ``WithResultSet`` with async
versions and adds async iteration and the async context manager protocol.
Synchronous iteration raises ``TypeError``. Subclasses override the fetch
methods with async versions.
Combines ``WithResultSet`` with ``AioBaseCursor`` and ``CursorIterator``,
and provides async ``executemany`` and ``cancel``, async iteration, and the
async context manager protocol. Synchronous iteration raises
``TypeError``. Subclasses implement the fetch methods as coroutines.

Subclasses override ``execute()`` and optionally ``__init__`` and
format-specific helpers.
Expand Down
4 changes: 2 additions & 2 deletions pyathena/arrow/cursor.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from pyathena.error import OperationalError, ProgrammingError
from pyathena.model import AthenaQueryExecution
from pyathena.options import ExecuteOptions
from pyathena.result_set import WithResultSet
from pyathena.result_set import WithFetch

if TYPE_CHECKING:
import polars as pl
Expand All @@ -22,7 +22,7 @@
_logger = logging.getLogger(__name__)


class ArrowCursor(WithResultSet):
class ArrowCursor(WithFetch):
"""Cursor for handling Apache Arrow Table results from Athena queries.

This cursor returns query results as Apache Arrow Tables, which provide
Expand Down
4 changes: 2 additions & 2 deletions pyathena/cursor.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,12 @@
from pyathena.error import OperationalError, ProgrammingError
from pyathena.model import AthenaQueryExecution
from pyathena.options import ExecuteOptions
from pyathena.result_set import AthenaDictResultSet, AthenaResultSet, WithResultSet
from pyathena.result_set import AthenaDictResultSet, AthenaResultSet, WithFetch

_logger = logging.getLogger(__name__)


class Cursor(WithResultSet):
class Cursor(WithFetch):
"""A DB API 2.0 compliant cursor for executing SQL queries on Amazon Athena.

The Cursor class provides methods for executing SQL queries against Amazon Athena
Expand Down
4 changes: 2 additions & 2 deletions pyathena/pandas/cursor.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,15 @@
DefaultPandasUnloadTypeConverter,
)
from pyathena.pandas.result_set import AthenaPandasResultSet, PandasDataFrameIterator
from pyathena.result_set import WithResultSet
from pyathena.result_set import WithFetch

if TYPE_CHECKING:
from pandas import DataFrame

_logger = logging.getLogger(__name__)


class PandasCursor(WithResultSet):
class PandasCursor(WithFetch):
"""Cursor for handling pandas DataFrame results from Athena queries.

This cursor returns query results as pandas DataFrames with memory-efficient
Expand Down
4 changes: 2 additions & 2 deletions pyathena/polars/cursor.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
DefaultPolarsUnloadTypeConverter,
)
from pyathena.polars.result_set import AthenaPolarsResultSet
from pyathena.result_set import WithResultSet
from pyathena.result_set import WithFetch

if TYPE_CHECKING:
import polars as pl
Expand All @@ -27,7 +27,7 @@
_logger = logging.getLogger(__name__)


class PolarsCursor(WithResultSet):
class PolarsCursor(WithFetch):
"""Cursor for handling Polars DataFrame results from Athena queries.

This cursor returns query results as Polars DataFrames using Polars' native
Expand Down
40 changes: 25 additions & 15 deletions pyathena/result_set.py
Original file line number Diff line number Diff line change
Expand Up @@ -794,14 +794,13 @@ def _get_rows(
]


class WithResultSet(BaseCursor, CursorIterator):
"""Base class of the SQL cursors that keep a result set.

Provides the result set and its properties, fetch, ``close``,
``executemany``, ``cancel``, and sync iteration. The sync SQL cursors
subclass it directly. For the asyncio cursors, ``WithAsyncFetch``
overrides ``executemany`` and ``cancel`` with async versions, and its
subclasses override the fetch methods.
class WithResultSet:
"""Mixin that keeps a cursor's query ID and result set.

Provides the query ID, the result set and its properties, ``arraysize``,
``rownumber``, ``rowcount``, and ``close``. ``WithFetch`` and
``WithAsyncFetch`` list it before ``BaseCursor`` / ``AioBaseCursor`` and
``CursorIterator``, so that these members take precedence over theirs.
"""

def __init__(self, arraysize: int | None = None, **kwargs) -> None:
Expand All @@ -811,7 +810,7 @@ def __init__(self, arraysize: int | None = None, **kwargs) -> None:
arraysize: Default number of rows per ``fetchmany()`` call,
validated by the ``arraysize`` setter. If None,
``DEFAULT_FETCH_SIZE`` is used.
**kwargs: Arguments passed to ``BaseCursor.__init__``.
**kwargs: Arguments passed to the next ``__init__`` in the MRO.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Independent review (relayed): FINDINGS → 1 finding, rejected after verification

Reviewer: Codex CLI 0.157.1, model gpt-6-sol, sandbox read-only, session 01a0f4da-98ac-7700-affd-ec00465f1dd0. Static review only (no lint/tests run by the reviewer, as instructed). Base 809b38070ff7ed69b960f9f4318a82e03723fd2c, head 85f0baa6b093adfc3b262d0f9a29dbdcd71ccb78; the review snapshot and the PR worktree were unchanged afterwards. The prompt contained the diff, the intended structure, and the pre-refactor commit for comparison, without the PR number, description, commit message, or self-review results.

Reviewer coverage: method resolution and initialization of the SQL cursor classes (incl. Dict variants), abstract methods, arraysize and lifecycle behavior, SQLAlchemy and Spark callers, changed docstrings, the new test, repository conventions. Reviewer: "no source-level method-resolution change or newly abstract concrete SQL cursor."

Finding (P2), pyathena/result_set.py:819: with no base beyond object, mypy would check super().__init__(**kwargs) against object.__init__ and just lint would fail.

Verification: rejected. uv run mypy . (inside just lint) reports no issues at this head, uv run mypy pyathena/result_set.py pyathena/aio/common.py succeeds, and the CI lint job passed at 85f0baa. mypy does not report unpacking a dict[str, Any] (**kwargs) into object.__init__, since it cannot know the mapping is non-empty. No change.


Raises:
ProgrammingError: If ``arraysize`` is outside the range the
Expand Down Expand Up @@ -1107,6 +1106,23 @@ def rownumber(self) -> int | None:
"""
return self.result_set.rownumber if self.result_set else None

def close(self) -> None:
"""Close the cursor and release associated resources."""
self._rowcount = -1
if self.result_set and not self.result_set.is_closed:
self.result_set.close()


class WithFetch(WithResultSet, BaseCursor, CursorIterator):

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Self-review round one (behavior and implementation): CLEAN

Base 809b38070ff7ed69b960f9f4318a82e03723fd2c, head 7d9ba8f55b82eae282567d61aae918ff797a595b.

Covered: pyathena/result_set.py (WithResultSet, WithFetch), pyathena/aio/common.py (WithAsyncFetch), the 5 sync cursor modules, tests/pyathena/test_result_set.py.

  • Behavior: for the 12 SQL cursors, every resolved member has the same instructions as on master; only the defining class of the sync fetch* / executemany / cancel moves to WithFetch. __abstractmethods__ is empty for all 12.
  • __init__ chain: aio/common.py defines no __init__, so WithResultSet.__init__ → BaseCursor.__init__ → CursorIterator.__init__ runs in the same order as on master. Instantiating all 12 cursors with and without arraysize=5000 gives identical instance attributes (names, order, values), arraysize / rowcount / rownumber / query_id / result_set, ProgrammingError for the capped cursors, and rowcount after close(), on master and this branch.
  • Callers: nothing in pyathena/ uses isinstance / issubclass with these classes, so WithResultSet no longer subclassing BaseCursor does not affect a code path.
  • Test quality: test_result_set.py fails on master (WithFetch import) and on a fold back into a base class (__bases__), and passes here.

No findings.

"""Base class of the sync SQL cursors.

Combines ``WithResultSet`` with ``BaseCursor`` and ``CursorIterator``, and
provides sync fetch, ``executemany``, ``cancel``, and sync iteration.

Subclasses override ``execute()`` and optionally ``__init__`` and
format-specific helpers.
"""

def fetchone(
self,
) -> tuple[Any | None, ...] | dict[Any, Any | None] | None:
Expand Down Expand Up @@ -1158,12 +1174,6 @@ def fetchall(
result_set = cast(AthenaResultSet, self.result_set)
return result_set.fetchall()

def close(self) -> None:
"""Close the cursor and release associated resources."""
self._rowcount = -1
if self.result_set and not self.result_set.is_closed:
self.result_set.close()

def executemany(
self,
operation: str,
Expand Down
4 changes: 2 additions & 2 deletions pyathena/s3fs/cursor.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,14 @@
from pyathena.error import OperationalError
from pyathena.model import AthenaQueryExecution
from pyathena.options import ExecuteOptions
from pyathena.result_set import WithResultSet
from pyathena.result_set import WithFetch
from pyathena.s3fs.converter import DefaultS3FSTypeConverter
from pyathena.s3fs.result_set import AthenaS3FSResultSet, CSVReaderType

_logger = logging.getLogger(__name__)


class S3FSCursor(WithResultSet):
class S3FSCursor(WithFetch):
"""Cursor for reading CSV results via S3FileSystem without pandas/pyarrow.

This cursor uses Python's standard csv module and PyAthena's S3FileSystem
Expand Down
24 changes: 24 additions & 0 deletions tests/pyathena/test_result_set.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# Copyright 2026 The PyAthena authors
#
# Licensed under the MIT License.
# See LICENSE or https://opensource.org/licenses/MIT.
#
# SPDX-License-Identifier: MIT
import pytest

from pyathena.aio.common import AioBaseCursor, WithAsyncFetch
from pyathena.common import BaseCursor, CursorIterator
from pyathena.result_set import WithFetch, WithResultSet


class TestWithResultSet:
def test_is_mixin(self):
assert WithResultSet.__bases__ == (object,)

@pytest.mark.parametrize(
("cursor_base", "base"),
[(WithFetch, BaseCursor), (WithAsyncFetch, AioBaseCursor)],
)
def test_precedes_cursor_bases(self, cursor_base, base):
# Listed first, so that its members take precedence over the cursor bases'.
assert cursor_base.__bases__ == (WithResultSet, base, CursorIterator)
Loading