From 0ecf23bc77d7b00d1da95732ba400bc3d4444f90 Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Mon, 28 Sep 2026 17:00:12 +0900 Subject: [PATCH 1/5] Stub every page in the listing resume test test_listing_resumes_after_a_failed_glue_request passed all but the injected ListTableMetadata request to Athena, so a real account-level metadata throttle changed the request sequence it asserts. Serve synthetic pages instead so the test only exercises the resume logic. Closes #842 Co-Authored-By: Claude Opus 5.5 --- tests/pyathena/test_cursor.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/tests/pyathena/test_cursor.py b/tests/pyathena/test_cursor.py index eb64f494a..3c0c63d19 100644 --- a/tests/pyathena/test_cursor.py +++ b/tests/pyathena/test_cursor.py @@ -1576,29 +1576,34 @@ def test_other_metadata_exception_keeps_its_retries(self, cursor, monkeypatch): ) def test_listing_resumes_after_a_failed_glue_request(self, cursor, monkeypatch): # A page throttled mid-listing is read again, not the pages before it. - expected = sorted(m.name for m in cursor.list_table_metadata(max_results=2)) - client = cursor.connection.client - list_table_metadata = client.list_table_metadata + # The pages are stubbed so that real throttling cannot change the requests. + pages = { + None: {"TableMetadataList": [{"Name": "t0"}, {"Name": "t1"}], "NextToken": "1"}, + "1": {"TableMetadataList": [{"Name": "t2"}, {"Name": "t3"}], "NextToken": "2"}, + "2": {"TableMetadataList": [{"Name": "t4"}]}, + } requests = [] - def throttle_second_page_once(**kwargs): + def throttle_second_request_once(**kwargs): requests.append(kwargs.get("NextToken")) if len(requests) == 2: raise ClientError( {"Error": {"Code": "ThrottlingException", "Message": "Rate exceeded"}}, "ListTableMetadata", ) - return list_table_metadata(**kwargs) + return pages[kwargs.get("NextToken")] - monkeypatch.setattr(client, "list_table_metadata", throttle_second_page_once) - self._unreachable_glue(cursor.connection, monkeypatch) + monkeypatch.setattr( + cursor.connection.client, "list_table_metadata", throttle_second_request_once + ) + glue = self._unreachable_glue(cursor.connection, monkeypatch) - assert sorted(m.name for m in cursor.list_table_metadata(max_results=2)) == expected + names = [m.name for m in cursor.list_table_metadata(max_results=2)] + + assert names == ["t0", "t1", "t2", "t3", "t4"] # Every page once, and the throttled second page a second time. - assert len(expected) > 2 - assert requests[0] is None - assert requests[1] == requests[2] - assert len(requests) == len(set(requests)) + 1 + assert requests == [None, "1", "1", "2"] + assert not glue.reachable @pytest.mark.parametrize( ("cursor", "catalog_name"), From 8b0d8a84917db31465cb3be521367849e28a4b67 Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Mon, 28 Sep 2026 21:39:28 +0900 Subject: [PATCH 2/5] Simplify the stubbed listing resume test Drop the max_results argument the stubbed pages ignore, read the request token once, and inline the listing assertion. Co-Authored-By: Claude Opus 5.5 --- tests/pyathena/test_cursor.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tests/pyathena/test_cursor.py b/tests/pyathena/test_cursor.py index 3c0c63d19..00a089516 100644 --- a/tests/pyathena/test_cursor.py +++ b/tests/pyathena/test_cursor.py @@ -1582,26 +1582,23 @@ def test_listing_resumes_after_a_failed_glue_request(self, cursor, monkeypatch): "1": {"TableMetadataList": [{"Name": "t2"}, {"Name": "t3"}], "NextToken": "2"}, "2": {"TableMetadataList": [{"Name": "t4"}]}, } + client = cursor.connection.client requests = [] def throttle_second_request_once(**kwargs): - requests.append(kwargs.get("NextToken")) + token = kwargs.get("NextToken") + requests.append(token) if len(requests) == 2: raise ClientError( {"Error": {"Code": "ThrottlingException", "Message": "Rate exceeded"}}, "ListTableMetadata", ) - return pages[kwargs.get("NextToken")] + return pages[token] - monkeypatch.setattr( - cursor.connection.client, "list_table_metadata", throttle_second_request_once - ) + monkeypatch.setattr(client, "list_table_metadata", throttle_second_request_once) glue = self._unreachable_glue(cursor.connection, monkeypatch) - names = [m.name for m in cursor.list_table_metadata(max_results=2)] - - assert names == ["t0", "t1", "t2", "t3", "t4"] - # Every page once, and the throttled second page a second time. + assert [m.name for m in cursor.list_table_metadata()] == ["t0", "t1", "t2", "t3", "t4"] assert requests == [None, "1", "1", "2"] assert not glue.reachable From 2a83e8d129820f6ebd18fbbc84c766373215fd52 Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Mon, 28 Sep 2026 21:48:29 +0900 Subject: [PATCH 3/5] Check that a listing reads every page against Athena With the resume test stubbed, no test followed a real NextToken: the session's 16 tables fit in one page of the default 50. List them five per page and check that every session table appears once. The filter keeps tables other tests create and drop out of the comparison, and a throttled page answered by Glue gives the same result. Co-Authored-By: Claude Opus 5.5 --- tests/pyathena/test_cursor.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/pyathena/test_cursor.py b/tests/pyathena/test_cursor.py index 00a089516..0323e51ff 100644 --- a/tests/pyathena/test_cursor.py +++ b/tests/pyathena/test_cursor.py @@ -1571,6 +1571,32 @@ def test_other_metadata_exception_keeps_its_retries(self, cursor, monkeypatch): cursor.get_table_metadata("one_row") assert calls == ["get_table_metadata"] * 2 + # The tables the test session creates; other tests add and drop their own. + _SESSION_TABLES = ( + "one_row", + "many_rows", + "one_row_complex", + "partition_table", + "integer_na_values", + "boolean_na_values", + "execute_many", + "execute_many_aio", + "execute_many_pandas", + "execute_many_pandas_unload_auto", + "execute_many_pandas_unload_pyarrow", + "execute_many_arrow", + "execute_many_arrow_unload", + "execute_many_polars", + "execute_many_polars_unload", + "parquet_with_compression", + ) + + def test_listing_reads_every_page(self, cursor): + # At five tables per page, the session's 16 tables span several pages. + names = [m.name for m in cursor.list_table_metadata(max_results=5)] + + assert sorted(n for n in names if n in self._SESSION_TABLES) == sorted(self._SESSION_TABLES) + @pytest.mark.parametrize( "cursor", [{"retry_config": RetryConfig(attempt=1)}], indirect=["cursor"] ) From 6cef032c4f5c1ee5a0809cc18346e4f14a3d88e0 Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Fri, 2 Oct 2026 01:04:05 +0900 Subject: [PATCH 4/5] Take the session's tables for the listing test from tests.pyathena.tables The session now creates its tables and views from TABLES and VIEWS, and the executemany tables are created per test, so the hardcoded list was stale. Read the names from the shared definitions, include the views that ListTableMetadata also returns, and use four per page so that they span several pages. Co-Authored-By: Claude Opus 5.5 --- tests/pyathena/test_cursor.py | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/tests/pyathena/test_cursor.py b/tests/pyathena/test_cursor.py index 0323e51ff..7b605c5af 100644 --- a/tests/pyathena/test_cursor.py +++ b/tests/pyathena/test_cursor.py @@ -36,6 +36,7 @@ from pyathena.util import RetryConfig from tests import ENV from tests.pyathena.conftest import connect +from tests.pyathena.tables import TABLES, VIEWS from tests.pyathena.util import succeeded_query_execution, throttle_metadata_api, unreachable_glue _logger = logging.getLogger(__name__) @@ -1571,31 +1572,13 @@ def test_other_metadata_exception_keeps_its_retries(self, cursor, monkeypatch): cursor.get_table_metadata("one_row") assert calls == ["get_table_metadata"] * 2 - # The tables the test session creates; other tests add and drop their own. - _SESSION_TABLES = ( - "one_row", - "many_rows", - "one_row_complex", - "partition_table", - "integer_na_values", - "boolean_na_values", - "execute_many", - "execute_many_aio", - "execute_many_pandas", - "execute_many_pandas_unload_auto", - "execute_many_pandas_unload_pyarrow", - "execute_many_arrow", - "execute_many_arrow_unload", - "execute_many_polars", - "execute_many_polars_unload", - "parquet_with_compression", - ) - def test_listing_reads_every_page(self, cursor): - # At five tables per page, the session's 16 tables span several pages. - names = [m.name for m in cursor.list_table_metadata(max_results=5)] + # Other tests add and drop their own tables, so only the session's are compared. + # At four per page, the session's tables and views span several pages. + session_tables = sorted(t.name for t in (*TABLES, *VIEWS)) + names = [m.name for m in cursor.list_table_metadata(max_results=4)] - assert sorted(n for n in names if n in self._SESSION_TABLES) == sorted(self._SESSION_TABLES) + assert sorted(n for n in names if n in session_tables) == session_tables @pytest.mark.parametrize( "cursor", [{"retry_config": RetryConfig(attempt=1)}], indirect=["cursor"] From 5041b7637378c8eb98a55507d3e9432d1c1bd6db Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Fri, 2 Oct 2026 01:11:51 +0900 Subject: [PATCH 5/5] Make the listing test read every page from Athena A throttled page was answered by Glue with the whole listing, and a listing that ignored max_results came back in one page; either way the test passed without following a NextToken. Turn the Glue fallback off so a throttled page is retried instead, and require at least three distinct request tokens. Repeated tokens from retries are counted once. Co-Authored-By: Claude Opus 5.5 --- tests/pyathena/test_cursor.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/pyathena/test_cursor.py b/tests/pyathena/test_cursor.py index 7b605c5af..5533efcbe 100644 --- a/tests/pyathena/test_cursor.py +++ b/tests/pyathena/test_cursor.py @@ -1572,13 +1572,26 @@ def test_other_metadata_exception_keeps_its_retries(self, cursor, monkeypatch): cursor.get_table_metadata("one_row") assert calls == ["get_table_metadata"] * 2 - def test_listing_reads_every_page(self, cursor): + # Without the Glue fallback, a throttled page is retried, so Athena serves every page. + @pytest.mark.parametrize("cursor", [{"glue_metadata_fallback": False}], indirect=["cursor"]) + def test_listing_reads_every_page(self, cursor, monkeypatch): + client = cursor.connection.client + list_table_metadata = client.list_table_metadata + tokens = [] + + def record_token(**kwargs): + tokens.append(kwargs.get("NextToken")) + return list_table_metadata(**kwargs) + + monkeypatch.setattr(client, "list_table_metadata", record_token) # Other tests add and drop their own tables, so only the session's are compared. - # At four per page, the session's tables and views span several pages. session_tables = sorted(t.name for t in (*TABLES, *VIEWS)) + names = [m.name for m in cursor.list_table_metadata(max_results=4)] assert sorted(n for n in names if n in session_tables) == session_tables + # At four per page, the session's tables and views span three pages or more. + assert len(set(tokens)) >= 3 @pytest.mark.parametrize( "cursor", [{"retry_config": RetryConfig(attempt=1)}], indirect=["cursor"]