From 9111820ebb6243c961ea4cedf1cdf7e248701209 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 29 Aug 2026 20:46:42 +0300 Subject: [PATCH 01/13] gh-155775: Do not consume the CAPABILITY response in imaplib (GH-155789) It can again be read with response('CAPABILITY') after LOGIN and AUTHENTICATE. The capabilities in the greeting are still consumed. Also document the capabilities attribute. Co-authored-by: Claude Opus 5 (1M context) --- Doc/library/imaplib.rst | 14 ++++++++++++++ Lib/imaplib.py | 11 ++++++++--- Lib/test/test_imaplib.py | 6 ++++++ .../2026-08-14-14-10-00.gh-issue-155775.Kq3vTx.rst | 4 ++++ 4 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-14-14-10-00.gh-issue-155775.Kq3vTx.rst diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 910b0f00c0e7de..745683673ccf50 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -907,6 +907,20 @@ An :class:`IMAP4` instance has the following methods: The following attributes are defined on instances of :class:`IMAP4`: +.. attribute:: IMAP4.capabilities + + A tuple of the capabilities advertised by the server, in upper case. + + It is set when the connection is established, + and refreshed after a successful :meth:`~IMAP4.login`, + :meth:`~IMAP4.authenticate` or :meth:`~IMAP4.starttls`, + because the server can advertise different capabilities + in different connection states. + + .. versionchanged:: 3.14.7 + Refreshed after :meth:`~IMAP4.login` and :meth:`~IMAP4.authenticate`. + + .. attribute:: IMAP4.PROTOCOL_VERSION The most recent supported protocol in the ``CAPABILITY`` response from the diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 24d3a27f21d2d1..977d6bdb1c3d86 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -366,7 +366,8 @@ def _connect(self): self._encoding, 'replace') raise self.error('invalid greeting: ' + greeting) - self._refresh_capabilities() + # The greeting is not a response to a command. + self._refresh_capabilities(consume=True) if __debug__: if self.debug >= 3: self._mesg('CAPABILITIES: %r' % (self.capabilities,)) @@ -1509,10 +1510,14 @@ def _get_capabilities(self): self.capabilities = tuple(dat.split()) - def _refresh_capabilities(self): + def _refresh_capabilities(self, consume=False): # Use a CAPABILITY response sent by the server, or ask for it. + # Unless it is consumed, the response can still be read with + # response('CAPABILITY'). if 'CAPABILITY' in self.untagged_responses: - dat = self.untagged_responses.pop('CAPABILITY')[-1] + dat = self.untagged_responses['CAPABILITY'][-1] + if consume: + del self.untagged_responses['CAPABILITY'] self.capabilities = tuple(str(dat, self._encoding).upper().split()) else: self._get_capabilities() diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index d97da398803681..da63da54e8ef73 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -1069,6 +1069,8 @@ def cmd_ENABLE(self, tag, args): client.login('user', 'pass') self.assertIn('ENABLE', client.capabilities) self.assertIn('UTF8=ACCEPT', client.capabilities) + self.assertEqual(client.response('CAPABILITY'), + ('CAPABILITY', [b'IMAP4rev1 ENABLE UTF8=ACCEPT'])) typ, _ = client.enable('UTF8=ACCEPT') self.assertEqual(typ, 'OK') @@ -1087,6 +1089,8 @@ def cmd_AUTHENTICATE(self, tag, args): self.assertNotIn('ENABLE', client.capabilities) client.authenticate('MYAUTH', lambda x: b'fake') self.assertIn('ENABLE', client.capabilities) + self.assertEqual(client.response('CAPABILITY'), + ('CAPABILITY', [b'IMAP4rev1 ENABLE'])) def test_greeting_capabilities(self): # Capabilities advertised in the greeting are used directly, @@ -1100,6 +1104,8 @@ def cmd_CAPABILITY(self, tag, args): client, server = self._setup(GreetingHandler) self.assertEqual(client.capabilities, ('IMAP4REV1', 'ENABLE')) self.assertFalse(getattr(server, 'capability_queried', False)) + # The greeting is not a response to a command, so it is consumed. + self.assertEqual(client.response('CAPABILITY'), ('CAPABILITY', [None])) def test_login_requery_capabilities(self): # If the server does not advertise capabilities after login, diff --git a/Misc/NEWS.d/next/Library/2026-08-14-14-10-00.gh-issue-155775.Kq3vTx.rst b/Misc/NEWS.d/next/Library/2026-08-14-14-10-00.gh-issue-155775.Kq3vTx.rst new file mode 100644 index 00000000000000..309421ae08240f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-14-14-10-00.gh-issue-155775.Kq3vTx.rst @@ -0,0 +1,4 @@ +Fix a regression in :mod:`imaplib` introduced in the fix for :gh:`63121`: +refreshing the capabilities after a successful :meth:`~imaplib.IMAP4.login` +or :meth:`~imaplib.IMAP4.authenticate` consumed the ``CAPABILITY`` response, +so it could no longer be read with :meth:`~imaplib.IMAP4.response`. From 72a88d71c042be97921057935b5dfe3a9bc16daa Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 29 Aug 2026 20:53:36 +0300 Subject: [PATCH 02/13] gh-83371: Fix deadlock when a Pool callback raises an exception (GH-155777) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The exception killed the thread which handles results, so that the pool hung forever. It is now the result of the job and is raised by AsyncResult.get(), with the original error as its context. Co-authored-by: Sindri Guðmundsson Co-authored-by: Thomas Grainger --- Lib/multiprocessing/pool.py | 66 +++++++++--- Lib/test/_test_multiprocessing.py | 102 ++++++++++++++++++ ...6-08-14-10-49-45.gh-issue-83371.Km4Rt7.rst | 6 ++ 3 files changed, 157 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-14-10-49-45.gh-issue-83371.Km4Rt7.rst diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index f50bcbe4451bea..ef9460ac0aa681 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -763,6 +763,16 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): self.terminate() +def _chain_context(exc, context): + 'Set context as the context of exc, avoiding a cycle.' + seen = {id(context)} + while exc is not None and id(exc) not in seen: + seen.add(id(exc)) + if exc.__context__ is None: + exc.__context__ = context + return + exc = exc.__context__ + # # Class whose instances are returned by `Pool.apply_async()` # @@ -800,13 +810,25 @@ def get(self, timeout=None): def _set(self, i, obj): self._success, self._value = obj - if self._callback and self._success: - self._callback(self._value) - if self._error_callback and not self._success: - self._error_callback(self._value) - self._event.set() - del self._cache[self._job] - self._pool = None + try: + if self._success: + if self._callback: + self._callback(self._value) + else: + if self._error_callback: + self._error_callback(self._value) + except BaseException as exc: + # A failed callback becomes the result of the job. If it + # propagated, it would kill the result handler thread. + if not self._success: + # do not lose the original error + _chain_context(exc, self._value) + self._success = False + self._value = exc + finally: + self._event.set() + del self._cache[self._job] + self._pool = None __class_getitem__ = classmethod(types.GenericAlias) @@ -837,11 +859,16 @@ def _set(self, i, success_result): if success and self._success: self._value[i*self._chunksize:(i+1)*self._chunksize] = result if self._number_left == 0: - if self._callback: - self._callback(self._value) - del self._cache[self._job] - self._event.set() - self._pool = None + try: + if self._callback: + self._callback(self._value) + except BaseException as exc: + self._success = False + self._value = exc + finally: + del self._cache[self._job] + self._event.set() + self._pool = None else: if not success and self._success: # only store first exception @@ -849,11 +876,16 @@ def _set(self, i, success_result): self._value = result if self._number_left == 0: # only consider the result ready once all jobs are done - if self._error_callback: - self._error_callback(self._value) - del self._cache[self._job] - self._event.set() - self._pool = None + try: + if self._error_callback: + self._error_callback(self._value) + except BaseException as exc: + _chain_context(exc, self._value) + self._value = exc + finally: + del self._cache[self._job] + self._event.set() + self._pool = None # # Class whose instances are returned by `Pool.imap()` diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index e5f618f5f2e84f..46ed8843fcd051 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -3474,12 +3474,114 @@ def test_resource_warning(self): pool = None support.gc_collect() +class CallbackError(Exception): pass + +class CallbackBaseException(BaseException): pass + def raising(): raise KeyError("key") +def raising_map(x): + raise KeyError("key") + +def reraise(exc): + raise exc + +def raise_with_context(exc): + try: + raise ZeroDivisionError + except ZeroDivisionError: + raise CallbackError('callback failed') + def unpickleable_result(): return lambda: 42 +class _TestPoolCallbackErrors(BaseTestCase): + ALLOWED_TYPES = ('processes', ) + + @staticmethod + def _raise(value): + raise CallbackError('callback failed') + + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + def test_apply_async_callback_raises(self): + with multiprocessing.Pool(1) as p: + res = p.apply_async(sqr, (7,), callback=self._raise) + with self.assertRaises(CallbackError): + res.get(support.SHORT_TIMEOUT) + # the pool is still usable + self.assertEqual(p.apply(sqr, (3,)), 9) + self.assertTrue(p._result_handler.is_alive()) + + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + def test_apply_async_callback_raises_base_exception(self): + def raise_base(value): + raise CallbackBaseException + with multiprocessing.Pool(1) as p: + res = p.apply_async(sqr, (7,), callback=raise_base) + with self.assertRaises(CallbackBaseException): + res.get(support.SHORT_TIMEOUT) + # the pool did not hang + self.assertEqual(p.apply(sqr, (3,)), 9) + + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + def test_apply_async_error_callback_raises(self): + with multiprocessing.Pool(1) as p: + res = p.apply_async(raising, error_callback=self._raise) + with self.assertRaises(CallbackError) as cm: + res.get(support.SHORT_TIMEOUT) + # the original error is not lost + self.assertIsInstance(cm.exception.__context__, KeyError) + self.assertEqual(p.apply(sqr, (3,)), 9) + + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + def test_apply_async_error_callback_reraises(self): + with multiprocessing.Pool(1) as p: + res = p.apply_async(raising, error_callback=reraise) + with self.assertRaises(KeyError) as cm: + res.get(support.SHORT_TIMEOUT) + # the error is not its own context + self.assertIsNone(cm.exception.__context__) + self.assertEqual(p.apply(sqr, (3,)), 9) + + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + def test_map_async_error_callback_reraises(self): + with multiprocessing.Pool(1) as p: + res = p.map_async(raising_map, [0], error_callback=reraise) + with self.assertRaises(KeyError) as cm: + res.get(support.SHORT_TIMEOUT) + self.assertIsNone(cm.exception.__context__) + self.assertEqual(p.apply(sqr, (3,)), 9) + + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + def test_apply_async_error_callback_raises_with_context(self): + # the original error is kept at the end of the context chain + with multiprocessing.Pool(1) as p: + res = p.apply_async(raising, error_callback=raise_with_context) + with self.assertRaises(CallbackError) as cm: + res.get(support.SHORT_TIMEOUT) + context = cm.exception.__context__ + self.assertIsInstance(context, ZeroDivisionError) + self.assertIsInstance(context.__context__, KeyError) + self.assertEqual(p.apply(sqr, (3,)), 9) + + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + def test_map_async_callback_raises(self): + with multiprocessing.Pool(1) as p: + res = p.map_async(sqr, list(range(3)), callback=self._raise) + with self.assertRaises(CallbackError): + res.get(support.SHORT_TIMEOUT) + self.assertEqual(p.apply(sqr, (3,)), 9) + + @warnings_helper.ignore_fork_in_thread_deprecation_warnings() + def test_map_async_error_callback_raises(self): + with multiprocessing.Pool(1) as p: + res = p.map_async(raising_map, [0], error_callback=self._raise) + with self.assertRaises(CallbackError) as cm: + res.get(support.SHORT_TIMEOUT) + self.assertIsInstance(cm.exception.__context__, KeyError) + self.assertEqual(p.apply(sqr, (3,)), 9) + class _TestPoolWorkerErrors(BaseTestCase): ALLOWED_TYPES = ('processes', ) diff --git a/Misc/NEWS.d/next/Library/2026-08-14-10-49-45.gh-issue-83371.Km4Rt7.rst b/Misc/NEWS.d/next/Library/2026-08-14-10-49-45.gh-issue-83371.Km4Rt7.rst new file mode 100644 index 00000000000000..ca30859c4a49c5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-14-10-49-45.gh-issue-83371.Km4Rt7.rst @@ -0,0 +1,6 @@ +Fix a deadlock in :class:`multiprocessing.pool.Pool` when *callback* or +*error_callback* raises an exception. +It killed the thread which handles results, so that the pool hung forever. +The exception is now the result of the job, +as an error raised while iterating the input, +and is raised by :meth:`!AsyncResult.get`. From 3e9efe550971a02a1580f2202399c1249d4be548 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Sat, 29 Aug 2026 18:55:23 +0100 Subject: [PATCH 03/13] gh-151365: Revert temporary Git downgrade (#156584) Revert "gh-151365: Temporarily downgrade Git to prevent fatal error (#155495)" This reverts commit 6d2b551399342fc6621195abfde7d79bcb1d7a4b. --- .github/workflows/reusable-check-html-ids.yml | 7 ------- .github/workflows/reusable-context.yml | 9 --------- .github/workflows/reusable-docs.yml | 8 -------- 3 files changed, 24 deletions(-) diff --git a/.github/workflows/reusable-check-html-ids.yml b/.github/workflows/reusable-check-html-ids.yml index bdebacc9eed06b..41ba1288be1ecf 100644 --- a/.github/workflows/reusable-check-html-ids.yml +++ b/.github/workflows/reusable-check-html-ids.yml @@ -20,13 +20,6 @@ jobs: with: persist-credentials: false ref: ${{ github.event.pull_request.head.sha }} - - name: 'Downgrade Git' - # Temporarily downgrade to 2.43 until 2.55 is in the runner image, - # to avoid "fatal: shallow file has changed since we read it" bug. - # See https://github.com/python/cpython/issues/151365. - run: | - sudo apt-get install -y --allow-downgrades 'git=1:2.43.*' 'git-man=1:2.43.*' - git --version - name: 'Find merge base' id: merge-base run: | diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml index f307326bae4120..c998cbff181dd1 100644 --- a/.github/workflows/reusable-context.yml +++ b/.github/workflows/reusable-context.yml @@ -90,15 +90,6 @@ jobs: || '' }} - - name: 'Downgrade Git' - # Temporarily downgrade to 2.43 until 2.55 is in the runner image, - # to avoid "fatal: shallow file has changed since we read it" bug. - # See https://github.com/python/cpython/issues/151365. - if: github.event_name == 'pull_request' - run: | - sudo apt-get install -y --allow-downgrades 'git=1:2.43.*' 'git-man=1:2.43.*' - git --version - # Adapted from https://github.com/actions/checkout/issues/520#issuecomment-1167205721 - name: Fetch commits to get branch diff if: github.event_name == 'pull_request' diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 6b1a9076c8251b..199e0fd8d181f0 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -47,14 +47,6 @@ jobs: && github.event.pull_request.head.sha || '' }} - - name: 'Downgrade Git' - # Temporarily downgrade to 2.43 until 2.55 is in the runner image, - # to avoid "fatal: shallow file has changed since we read it" bug. - # See https://github.com/python/cpython/issues/151365. - if: github.event_name == 'pull_request' - run: | - sudo apt-get install -y --allow-downgrades 'git=1:2.43.*' 'git-man=1:2.43.*' - git --version # Adapted from https://github.com/actions/checkout/issues/520#issuecomment-1167205721 - name: 'Fetch commits to get branch diff' if: github.event_name == 'pull_request' From 0e8100a140414ceb9086a1d5d7dd24209f111d41 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 29 Aug 2026 21:04:01 +0300 Subject: [PATCH 04/13] gh-85470: Delay load ole32.dll in the _ctypes module (GH-155435) ole32.dll pulls in user32.dll, which makes the system treat the process as a GUI process and stop sending logoff and shutdown console control events to it. It is only needed for ProgIDFromCLSID() when formatting a COM error. --- .../Windows/2026-08-09-15-40-00.gh-issue-85470.Lm4Xt9.rst | 4 ++++ PCbuild/_ctypes.vcxproj | 1 + 2 files changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Windows/2026-08-09-15-40-00.gh-issue-85470.Lm4Xt9.rst diff --git a/Misc/NEWS.d/next/Windows/2026-08-09-15-40-00.gh-issue-85470.Lm4Xt9.rst b/Misc/NEWS.d/next/Windows/2026-08-09-15-40-00.gh-issue-85470.Lm4Xt9.rst new file mode 100644 index 00000000000000..1d623c890e960b --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-08-09-15-40-00.gh-issue-85470.Lm4Xt9.rst @@ -0,0 +1,4 @@ +Importing :mod:`ctypes` on Windows no longer loads ``user32.dll``, which +made the system treat the process as a GUI process and stop sending logoff +and shutdown console control events to it. ``ole32.dll``, which pulled +``user32.dll`` in, is now delay loaded. diff --git a/PCbuild/_ctypes.vcxproj b/PCbuild/_ctypes.vcxproj index 63d5fa49cd4e17..e56d812d78dcd8 100644 --- a/PCbuild/_ctypes.vcxproj +++ b/PCbuild/_ctypes.vcxproj @@ -98,6 +98,7 @@ /EXPORT:DllGetClassObject,PRIVATE /EXPORT:DllCanUnloadNow,PRIVATE %(AdditionalOptions) + ole32.dll From c922aa0ddd264395cd606fdfd20512fa54108cfb Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 29 Aug 2026 21:05:33 +0300 Subject: [PATCH 05/13] gh-72353: Continue reading from the console after ignored Ctrl+C (GH-155434) If the console cancels a read because of Ctrl+C, but the SIGINT handler does not raise an exception, restart the read instead of ending it as if at end of file. Co-authored-by: Valeriya Sinevich Co-authored-by: Zackery Spytz --- .../2026-08-09-14-20-00.gh-issue-72353.Vw7Kq2.rst | 3 +++ Modules/_io/winconsoleio.c | 11 +++++++++-- Parser/myreadline.c | 12 +++++++++--- 3 files changed, 21 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2026-08-09-14-20-00.gh-issue-72353.Vw7Kq2.rst diff --git a/Misc/NEWS.d/next/Windows/2026-08-09-14-20-00.gh-issue-72353.Vw7Kq2.rst b/Misc/NEWS.d/next/Windows/2026-08-09-14-20-00.gh-issue-72353.Vw7Kq2.rst new file mode 100644 index 00000000000000..95c771823ab77f --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-08-09-14-20-00.gh-issue-72353.Vw7Kq2.rst @@ -0,0 +1,3 @@ +Reading from the console on Windows now continues if the read was cancelled +by Ctrl+C, but the :const:`~signal.SIGINT` handler did not raise an +exception. Previously the read ended as if at end of file. diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 4cd71094e8f459..e96c5bd738fe0a 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -644,8 +644,8 @@ read_console_w(HANDLE handle, DWORD maxlen, DWORD *readlen) { break; err = 0; HANDLE hInterruptEvent = _PyOS_SigintEvent(); - if (WaitForSingleObjectEx(hInterruptEvent, 100, FALSE) - == WAIT_OBJECT_0) { + DWORD state = WaitForSingleObjectEx(hInterruptEvent, 100, FALSE); + if (state == WAIT_OBJECT_0) { ResetEvent(hInterruptEvent); Py_BLOCK_THREADS sig = PyErr_CheckSignals(); @@ -653,6 +653,13 @@ read_console_w(HANDLE handle, DWORD maxlen, DWORD *readlen) { if (sig < 0) break; } + else if (state != WAIT_TIMEOUT) { + err = GetLastError(); + break; + } + /* The console cancelled the read and flushed its input buffer, + but no exception was raised. Start the read over. */ + continue; } *readlen += n; diff --git a/Parser/myreadline.c b/Parser/myreadline.c index 457b9598a0e61a..339f55afa0dcfa 100644 --- a/Parser/myreadline.c +++ b/Parser/myreadline.c @@ -162,8 +162,8 @@ _PyOS_WindowsConsoleReadline(PyThreadState *tstate, HANDLE hStdIn) goto exit; err = 0; HANDLE hInterruptEvent = _PyOS_SigintEvent(); - if (WaitForSingleObjectEx(hInterruptEvent, 100, FALSE) - == WAIT_OBJECT_0) { + DWORD state = WaitForSingleObjectEx(hInterruptEvent, 100, FALSE); + if (state == WAIT_OBJECT_0) { ResetEvent(hInterruptEvent); PyEval_RestoreThread(tstate); s = PyErr_CheckSignals(); @@ -172,7 +172,13 @@ _PyOS_WindowsConsoleReadline(PyThreadState *tstate, HANDLE hStdIn) goto exit; } } - break; + else if (state != WAIT_TIMEOUT) { + err = GetLastError(); + goto exit; + } + /* The console cancelled the read and flushed its input buffer, + but no exception was raised. Start the read over. */ + continue; } total_read += n_read; From b4aea41d1cd0da1f34e42366d64e9e19d557eb87 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 29 Aug 2026 21:15:25 +0300 Subject: [PATCH 06/13] gh-87587: Fix os.device_encoding() for any console on Windows (GH-155410) It was hard coded to map file descriptors 0, 1 and 2 to the console code page, but a console can be opened as any file descriptor, and other character devices, like NUL, are not consoles. The file handle is now queried. The UTF-8 code page is also now reported as "utf-8" instead of "cp65001". --- Lib/test/test_os/test_windows.py | 38 ++++++++++++ ...6-08-09-04-00-00.gh-issue-87587.devenc.rst | 4 ++ Python/fileutils.c | 58 ++++++++++++++----- 3 files changed, 84 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2026-08-09-04-00-00.gh-issue-87587.devenc.rst diff --git a/Lib/test/test_os/test_windows.py b/Lib/test/test_os/test_windows.py index b21dd8a4dca660..3fb8454cddfde9 100644 --- a/Lib/test/test_os/test_windows.py +++ b/Lib/test/test_os/test_windows.py @@ -608,5 +608,43 @@ def cleanup(): self.assertGreaterEqual(stat1.st_atime, stat2.st_atime) +class Win32DeviceEncodingTests(unittest.TestCase): + # gh-87587: any console file descriptor is supported, not only 0, 1 and 2, + # and other character devices are not consoles. + + @staticmethod + def expected_encoding(cp): + return 'utf-8' if cp == 65001 else 'cp%d' % cp + + def test_console(self): + import ctypes + kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) + try: + fin = open('CONIN$') + except OSError: + self.skipTest('no console') + with fin: + self.assertEqual(os.device_encoding(fin.fileno()), + self.expected_encoding(kernel32.GetConsoleCP())) + with open('CONOUT$', 'w') as fout: + self.assertEqual( + os.device_encoding(fout.fileno()), + self.expected_encoding(kernel32.GetConsoleOutputCP())) + + def test_not_a_console(self): + with open('NUL', 'w') as f: + self.assertTrue(os.isatty(f.fileno())) + self.assertIsNone(os.device_encoding(f.fileno())) + # Not a console even if it is a standard file descriptor. + saved = os.dup(1) + try: + os.dup2(f.fileno(), 1) + encoding = os.device_encoding(1) + finally: + os.dup2(saved, 1) + os.close(saved) + self.assertIsNone(encoding) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Windows/2026-08-09-04-00-00.gh-issue-87587.devenc.rst b/Misc/NEWS.d/next/Windows/2026-08-09-04-00-00.gh-issue-87587.devenc.rst new file mode 100644 index 00000000000000..06bba3e1f3d855 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-08-09-04-00-00.gh-issue-87587.devenc.rst @@ -0,0 +1,4 @@ +:func:`os.device_encoding` on Windows now returns the code page of any +console file descriptor, not only 0, 1 and 2, and returns ``None`` for other +character devices like ``NUL``. The UTF-8 code page is now reported as +``"utf-8"`` instead of ``"cp65001"``. diff --git a/Python/fileutils.c b/Python/fileutils.c index 5cbfd8e6ce4fa0..404fec83385e97 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -78,34 +78,59 @@ get_surrogateescape(_Py_error_handler errors, int *surrogateescape) PyObject * _Py_device_encoding(int fd) { - int valid; - Py_BEGIN_ALLOW_THREADS +#if defined(MS_WINDOWS) && defined(HAVE_WINDOWS_CONSOLE_IO) + HANDLE handle; + DWORD temp; + UINT cp = 0; + _Py_BEGIN_SUPPRESS_IPH - valid = isatty(fd); + handle = (HANDLE)_get_osfhandle(fd); _Py_END_SUPPRESS_IPH - Py_END_ALLOW_THREADS - if (!valid) + if (handle == INVALID_HANDLE_VALUE) { Py_RETURN_NONE; + } + + Py_BEGIN_ALLOW_THREADS + if (GetFileType(handle) == FILE_TYPE_CHAR) { + /* GetConsoleMode() only succeeds for a console handle. */ + if (!GetConsoleMode(handle, &temp)) { + /* Assume that access denied implies an output handle. */ + if (GetLastError() == ERROR_ACCESS_DENIED) { + cp = GetConsoleOutputCP(); + } + } + else if (GetNumberOfConsoleInputEvents(handle, &temp)) { + cp = GetConsoleCP(); + } + else { + cp = GetConsoleOutputCP(); + } + } + Py_END_ALLOW_THREADS -#ifdef MS_WINDOWS -#ifdef HAVE_WINDOWS_CONSOLE_IO - UINT cp; - if (fd == 0) - cp = GetConsoleCP(); - else if (fd == 1 || fd == 2) - cp = GetConsoleOutputCP(); - else - cp = 0; /* GetConsoleCP() and GetConsoleOutputCP() return 0 if the application has no console */ + if (cp == CP_UTF8) { + _Py_DECLARE_STR(utf_8, "utf-8"); + return &_Py_STR(utf_8); + } if (cp == 0) { Py_RETURN_NONE; } - return PyUnicode_FromFormat("cp%u", (unsigned int)cp); #else + int valid; + Py_BEGIN_ALLOW_THREADS + _Py_BEGIN_SUPPRESS_IPH + valid = isatty(fd); + _Py_END_SUPPRESS_IPH + Py_END_ALLOW_THREADS + if (!valid) { + Py_RETURN_NONE; + } + +#ifdef MS_WINDOWS Py_RETURN_NONE; -#endif /* HAVE_WINDOWS_CONSOLE_IO */ #else if (_PyRuntime.preconfig.utf8_mode) { _Py_DECLARE_STR(utf_8, "utf-8"); @@ -113,6 +138,7 @@ _Py_device_encoding(int fd) } return _Py_GetLocaleEncodingObject(); #endif +#endif /* MS_WINDOWS && HAVE_WINDOWS_CONSOLE_IO */ } From 0c38e057d0279c10a8cd7fd916ba1fa3c767170a Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 29 Aug 2026 21:17:47 +0300 Subject: [PATCH 07/13] gh-102475: Fix os.path.realpath() for names which look like a drive (GH-155399) The unresolved part of the path was joined with the resolved part using join(), so a component containing a colon (e.g. "spam:eggs") reset the path. It is now simply appended. A path relative to a drive which does not exist (e.g. "Z:spam") is now resolved against the root directory of that drive, as the Windows path normalization does. --- Lib/ntpath.py | 25 +++++++++++----- Lib/test/test_ntpath.py | 29 +++++++++++++++++++ ...-08-09-01-30-00.gh-issue-102475.rpjoin.rst | 4 +++ 3 files changed, 51 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-09-01-30-00.gh-issue-102475.rpjoin.rst diff --git a/Lib/ntpath.py b/Lib/ntpath.py index b3c23f0abc2d88..071941d3676712 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -626,12 +626,23 @@ def _getfinalpathname_nonstrict(path, ignored_error=OSError): allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 53, 65, 67, 87, 123, 161, 1005, 1920, 1921 # Non-strict algorithm is to find as much of the target directory - # as we can and join the rest. + # as we can and join the rest. join() is not used, because the tail + # can contain a colon and be mistaken for a drive (gh-102475). + if isinstance(path, bytes): + sep = b'\\' + else: + sep = '\\' + + def join(path, tail): + if path[-1:] == sep or not tail: + return path + tail + return path + sep + tail + tail = path[:0] while path: try: path = _getfinalpathname(path) - return join(path, tail) if tail else path + return join(path, tail) except ignored_error as ex: if ex.winerror not in allowed_winerror: raise @@ -642,7 +653,7 @@ def _getfinalpathname_nonstrict(path, ignored_error=OSError): new_path = _readlink_deep(path, ignored_error=ignored_error) if new_path != path: - return join(new_path, tail) if tail else new_path + return join(new_path, tail) except ignored_error: # If we fail to readlink(), let's keep traversing pass @@ -657,7 +668,7 @@ def _getfinalpathname_nonstrict(path, ignored_error=OSError): path, name = split(path) if path and not name: return path + tail - tail = join(name, tail) if tail else name + tail = join(name, tail) return tail def realpath(path, /, *, strict=False): @@ -666,7 +677,6 @@ def realpath(path, /, *, strict=False): prefix = b'\\\\?\\' unc_prefix = b'\\\\?\\UNC\\' new_unc_prefix = b'\\\\' - cwd = os.getcwdb() # bpo-38081: Special case for realpath(b'nul') devnull = b'nul' if normcase(path) == devnull: @@ -675,7 +685,6 @@ def realpath(path, /, *, strict=False): prefix = '\\\\?\\' unc_prefix = '\\\\?\\UNC\\' new_unc_prefix = '\\\\' - cwd = os.getcwd() # bpo-38081: Special case for realpath('nul') devnull = 'nul' if normcase(path) == devnull: @@ -692,7 +701,9 @@ def realpath(path, /, *, strict=False): ignored_error = OSError if not had_prefix and not isabs(path): - path = join(cwd, path) + # abspath() is used instead of join(cwd, path), because the path + # can be relative to another drive (gh-102475). + path = abspath(path) try: path = _getfinalpathname(path) initial_winerror = 0 diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 936332bf94ffe7..fcf98ad88a0f95 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -1535,6 +1535,35 @@ def test_isjunction(self): self.assertFalse(ntpath.isjunction('tmpdir')) self.assertPathEqual(ntpath.realpath('testjunc'), ntpath.realpath('tmpdir')) + @unittest.skipIf(sys.platform != 'win32', "Can only test on win32.") + def test_realpath_drive_like_names(self): + # gh-102475: the unresolved tail is appended, not joined, so a name + # which looks like a drive does not reset the path. + drive = ntpath.splitroot(os.getcwd())[0] + for path, expected in [ + ('C:/spam:eggs', 'C:\\spam:eggs'), + ('C:/nonexistent/spam:eggs', 'C:\\nonexistent\\spam:eggs'), + ('C:/spam:eggs/ham', 'C:\\spam:eggs\\ham'), + ('C:/nonexistent/spam:eggs/ham', 'C:\\nonexistent\\spam:eggs\\ham'), + ]: + with self.subTest(path=path): + self.assertEqual(ntpath.realpath(path), expected) + self.assertEqual(ntpath.realpath(os.fsencode(path)), + os.fsencode(expected)) + + @unittest.skipIf(sys.platform != 'win32', "Can only test on win32.") + def test_realpath_drive_relative(self): + # gh-102475: the working directory of a drive which does not exist + # is its root directory. + for drive in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ': + if not ntpath.exists(drive + ':'): + break + else: + raise unittest.SkipTest('all drives exist') + self.assertEqual(ntpath.realpath(drive + ':spam'), + drive + ':\\spam') + self.assertEqual(ntpath.realpath(drive + ':'), drive + ':\\') + def test_isfile_invalid_paths(self): isfile = ntpath.isfile self.assertIs(isfile('/tmp\udfffabcds'), False) diff --git a/Misc/NEWS.d/next/Library/2026-08-09-01-30-00.gh-issue-102475.rpjoin.rst b/Misc/NEWS.d/next/Library/2026-08-09-01-30-00.gh-issue-102475.rpjoin.rst new file mode 100644 index 00000000000000..fb647720791dac --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-09-01-30-00.gh-issue-102475.rpjoin.rst @@ -0,0 +1,4 @@ +Fix :func:`os.path.realpath` on Windows: the unresolved part of the path is +now appended, not joined, so a file name which looks like a drive (e.g. +``spam:eggs``) no longer discards the resolved part. A path relative to +another drive is now resolved against the root directory of that drive. From 0bb312a4eacc5df4d5556f61694416644fd6902b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 29 Aug 2026 22:17:43 +0300 Subject: [PATCH 08/13] gh-86427: Use the console code page for the stdio encoding on Windows (GH-155415) In the legacy Windows stdio mode PyConfig.stdio_encoding is now initialized with the console output code page instead of the ANSI code page. Only one encoding can be specified, so sys.stdin.encoding is still wrong if the input code page differs from the output one. --- Lib/test/test_cmd_line.py | 35 +++++++++++++++++++ ...8-09-07-00-00.gh-issue-86427.consolecp.rst | 3 ++ Python/initconfig.c | 26 ++++++++++++-- 3 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2026-08-09-07-00-00.gh-issue-86427.consolecp.rst diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 555cebeadc6053..2200770b4a45f0 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -1067,6 +1067,41 @@ def test_python_legacy_windows_stdio(self): support.skip_on_low_desktop_heap_memory_subprocess(p.returncode) self.assertEqual(p.returncode, 0) + @unittest.skipUnless(support.MS_WINDOWS, 'Test only applicable on Windows') + def test_python_legacy_windows_stdio_encoding(self): + # gh-86427: In the legacy mode the encoding of the standard streams + # is the encoding of the console. + import ctypes + kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) + try: + fin = open('CONIN$') + except OSError: + self.skipTest('no console') + # We cannot use PIPE, because the standard streams should be + # connected to the console. So we use the exit code. + code = ("import sys; sys.exit(sys.stdin.encoding != 'cp850' or " + "sys.stdout.encoding != 'cp850')") + env = os.environ.copy() + env['PYTHONLEGACYWINDOWSSTDIO'] = '1' + env['PYTHONUTF8'] = '0' + env.pop('PYTHONIOENCODING', None) + old_cp = kernel32.GetConsoleCP() + old_output_cp = kernel32.GetConsoleOutputCP() + with fin, open('CONOUT$', 'w') as fout: + try: + if not kernel32.SetConsoleCP(850): + self.skipTest('cannot set the console input code page') + if not kernel32.SetConsoleOutputCP(850): + self.skipTest('cannot set the console output code page') + proc = subprocess.run([sys.executable, '-c', code], env=env, + stdin=fin, stdout=fout, + stderr=subprocess.DEVNULL) + finally: + kernel32.SetConsoleCP(old_cp) + kernel32.SetConsoleOutputCP(old_output_cp) + support.skip_on_low_desktop_heap_memory_subprocess(proc.returncode) + self.assertEqual(proc.returncode, 0) + @unittest.skipIf("-fsanitize" in sysconfig.get_config_vars().get('PY_CFLAGS', ()), "PYTHONMALLOCSTATS doesn't work with ASAN") def test_python_malloc_stats(self): diff --git a/Misc/NEWS.d/next/Windows/2026-08-09-07-00-00.gh-issue-86427.consolecp.rst b/Misc/NEWS.d/next/Windows/2026-08-09-07-00-00.gh-issue-86427.consolecp.rst new file mode 100644 index 00000000000000..1cbf188c12e407 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-08-09-07-00-00.gh-issue-86427.consolecp.rst @@ -0,0 +1,3 @@ +Fix the encoding of the standard streams in the legacy Windows stdio mode +(:envvar:`PYTHONLEGACYWINDOWSSTDIO`). It is now the code page of the +console, as in Python 3.7, not the ANSI code page. diff --git a/Python/initconfig.c b/Python/initconfig.c index b9bacb17a66454..69d47a5872f5e3 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -2710,8 +2710,30 @@ config_init_stdio_encoding(PyConfig *config, /* Choose the default error handler based on the current locale. */ if (config->stdio_encoding == NULL) { - status = config_get_locale_encoding(config, preconfig, - &config->stdio_encoding); +#ifdef MS_WINDOWS + /* gh-86427: use the console code page. Only one encoding can be + specified, so the output code page is used: it affects two + streams of three. */ + UINT cp = config->legacy_windows_stdio ? GetConsoleOutputCP() : 0; + if (cp != 0) { + if (cp == CP_UTF8) { + status = PyConfig_SetString(config, &config->stdio_encoding, + L"utf-8"); + } + else { + wchar_t encoding[20]; + swprintf(encoding, Py_ARRAY_LENGTH(encoding), L"cp%u", + (unsigned int)cp); + status = PyConfig_SetString(config, &config->stdio_encoding, + encoding); + } + } + else +#endif + { + status = config_get_locale_encoding(config, preconfig, + &config->stdio_encoding); + } if (_PyStatus_EXCEPTION(status)) { return status; } From 55eef52e801be5fafa76c26b96a84ee737741dd1 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 29 Aug 2026 22:27:09 +0300 Subject: [PATCH 09/13] gh-108885: Report the examples of a doctest run by unittest (GH-155377) DocTestCase ran its examples with verbose=False, so there was no way to ask unittest for the details which doctest reports on its own. It now takes the verbosity from the test result, and reports every example if the test runner is asked for more than the test names. They are written to the stream of the test runner, so that they are not lost when it buffers the output of the test. To make this reachable: * unittest.TestResult has now a verbosity attribute, which it accepted but ignored. The test runner sets it, because a result class is free to filter what its constructor gets. * The unittest -v option is now counted, so that -vv means 3. * The verbosity of regrtest is one less, so it is translated where the test runner is created: -v reports the test names, as before, and -vv reports also the examples. Co-authored-by: Claude Opus 5 (1M context) --- Doc/library/doctest.rst | 6 ++ Doc/library/test.rst | 7 ++- Doc/library/unittest.rst | 18 +++++- Lib/doctest.py | 41 ++++++++++---- Lib/test/libregrtest/testresult.py | 6 +- Lib/test/test_doctest/test_doctest.py | 56 ++++++++++++++++++- Lib/test/test_regrtest.py | 12 ++++ Lib/test/test_unittest/test_discovery.py | 2 +- Lib/test/test_unittest/test_program.py | 26 +++++++++ Lib/test/test_unittest/test_result.py | 7 +++ Lib/test/test_unittest/test_runner.py | 29 ++++++++++ Lib/unittest/main.py | 7 ++- Lib/unittest/result.py | 3 + Lib/unittest/runner.py | 3 + ...-08-08-09-14-22.gh-issue-108885.C4ktfw.rst | 6 ++ ...-08-08-09-15-08.gh-issue-108885.Qm3vLp.rst | 2 + 16 files changed, 211 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-08-09-14-22.gh-issue-108885.C4ktfw.rst create mode 100644 Misc/NEWS.d/next/Tests/2026-08-08-09-15-08.gh-issue-108885.Qm3vLp.rst diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 3298697af8511b..4b61c07919dd02 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -1162,6 +1162,12 @@ from text files and modules with doctests: .. versionchanged:: 3.15 Run each example as a :ref:`subtest `. + .. versionchanged:: next + Report every example, as in verbose mode, if the test runner reports + more than the test names, i.e. its + :attr:`~unittest.TestResult.verbosity` is 3 or higher (for example + with ``python -m unittest -vv``). + Under the covers, :func:`DocTestSuite` creates a :class:`unittest.TestSuite` out of :class:`!doctest.DocTestCase` instances, and :class:`!DocTestCase` is a subclass of :class:`unittest.TestCase`. :class:`!DocTestCase` isn't documented diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 72ffc2a0b760ea..765811eba32fd4 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -232,9 +232,10 @@ The :mod:`!test.support` module defines the following constants: .. data:: verbose - ``True`` when verbose output is enabled. Should be checked when more - detailed information is desired about a running test. *verbose* is set by - :mod:`test.regrtest`. + How verbose the output is: the number of :option:`!-v` options which + :mod:`test.regrtest` was run with, and therefore ``0`` when verbose output + is not enabled. Should be checked when more detailed information is + desired about a running test. .. data:: is_jython diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 7afcdb368a3562..e1bc32fb79c2eb 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -180,6 +180,9 @@ You can run tests with more detail (higher verbosity) by passing in the -v flag: python -m unittest -v test_module +Repeat it for even more detail: ``-vv`` reports also the individual examples +of a :mod:`doctest`. + When executed without arguments :ref:`unittest-test-discovery` is started:: python -m unittest @@ -291,7 +294,11 @@ The ``discover`` sub-command has the following options: .. option:: -v, --verbose - Verbose output + Verbose output. May be repeated: ``-vv`` reports also the individual + examples of a :mod:`doctest`. + + .. versionchanged:: next + The option can be repeated. .. option:: -s, --start-directory directory @@ -2149,6 +2156,15 @@ Loading and running tests .. versionadded:: 3.5 + .. attribute:: verbosity + + The level of details which the test runner reports: ``0`` -- quiet, + ``1`` -- progress dots, ``2`` -- test names, ``3`` -- also the + individual examples of a :mod:`doctest`. A test runner is expected to + set it to its own verbosity. + + .. versionadded:: next + .. method:: wasSuccessful() Return ``True`` if all tests run so far have passed, otherwise returns diff --git a/Lib/doctest.py b/Lib/doctest.py index d5541ab41f38d4..a8f08868cf4957 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1201,6 +1201,18 @@ def _find_lineno(self, obj, source_lines): ## 5. DocTest Runner ###################################################################### +def _make_output_function(stream): + """Return a function writing to *stream*, whatever it can encode.""" + encoding = getattr(stream, 'encoding', None) + if encoding is None or encoding.lower() == 'utf-8': + return stream.write + def out(s): + # Use backslashreplace error handling on write + s = str(s.encode(encoding, 'backslashreplace'), encoding) + stream.write(s) + return out + + class DocTestRunner: """ A class used to run DocTest test cases, and accumulate statistics. @@ -1561,14 +1573,7 @@ def run(self, test, compileflags=None, out=None, clear_globs=True): save_stdout = sys.stdout if out is None: - encoding = save_stdout.encoding - if encoding is None or encoding.lower() == 'utf-8': - out = save_stdout.write - else: - # Use backslashreplace error handling on write - def out(s): - s = str(s.encode(encoding, 'backslashreplace'), encoding) - save_stdout.write(s) + out = _make_output_function(save_stdout) sys.stdout = self._fakeout # Patch pdb.set_trace to restore sys.stdout during interactive @@ -2322,6 +2327,9 @@ def report_skip(self, out, test, example): unittest.case._addSkip(self._test_result, self._subTest(), '') def report_success(self, out, test, example, got): + # Report "ok" if verbose, to close what report_start() opened. A + # failed or skipped example is reported by the test result instead. + super().report_success(out, test, example, got) self._test_result.addSubTest(self._test_case, self._subTest(), None) def report_unexpected_exception(self, out, test, example, exc_info): @@ -2401,10 +2409,23 @@ def runTest(self): if getattr(result, 'failfast', False): optionflags |= FAIL_FAST + # Report every example only if the test runner is asked for more than + # the test names it reports at verbosity 2. Write them to its stream, + # so that they are not swallowed by result.buffer. + verbose = getattr(result, 'verbosity', 1) >= 3 + stream = getattr(result, 'stream', None) + out = None + if verbose and stream is not None: + out = _make_output_function(stream) + if test.examples and not getattr(result, '_newline', True): + # End the line which startTest() left open. + out('\n') + result._newline = True + runner = _DocTestCaseRunner(optionflags=optionflags, - checker=self._dt_checker, verbose=False, + checker=self._dt_checker, verbose=verbose, test_case=self, test_result=result) - results = runner.run(test, clear_globs=False) + results = runner.run(test, out=out, clear_globs=False) if results.skipped == results.attempted: raise unittest.SkipTest("all examples were skipped") diff --git a/Lib/test/libregrtest/testresult.py b/Lib/test/libregrtest/testresult.py index 1820f354572521..605f1f4e6a89fb 100644 --- a/Lib/test/libregrtest/testresult.py +++ b/Lib/test/libregrtest/testresult.py @@ -16,7 +16,7 @@ class RegressionTestResult(unittest.TextTestResult): def __init__(self, stream, descriptions, verbosity): super().__init__(stream=stream, descriptions=descriptions, - verbosity=2 if verbosity else 0) + verbosity=verbosity) self.buffer = True if self.USE_XML: from xml.etree import ElementTree as ET @@ -150,10 +150,12 @@ def run(self, test): def get_test_runner_class(verbosity, buffer=False): if verbosity: + # The verbosity of regrtest is one less than the verbosity of + # unittest: -v reports the test names, -vv also the doctest examples. return functools.partial(unittest.TextTestRunner, resultclass=RegressionTestResult, buffer=buffer, - verbosity=verbosity) + verbosity=verbosity + 1) return functools.partial(QuietRegressionTestRunner, buffer=buffer) def get_test_runner(stream, verbosity, capture_output=False): diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index b125693ab0891c..776ad83ee6d6ce 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -6,6 +6,7 @@ from test.support import import_helper import doctest import functools +import io import os import sys import importlib @@ -469,7 +470,7 @@ def basics(): r""" >>> tests = finder.find(sample_func) >>> print(tests) # doctest: +ELLIPSIS - [] + [] The exact name depends on how test_doctest was invoked, so allow for leading path components. @@ -803,6 +804,59 @@ def myfunc(): self.assertEqual((x, y), (2, 3)) +class TestDocTestSuiteVerbosity(unittest.TestCase): + + def run_suite(self, module='test.test_doctest.sample_doctest', **kwargs): + """Return what the test runner wrote and what leaked to stdout.""" + suite = doctest.DocTestSuite(module) + stream = io.StringIO() + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + unittest.TextTestRunner(stream=stream, **kwargs).run(suite) + return stream.getvalue(), stdout.getvalue() + + def test_quiet(self): + for verbosity in range(3): + with self.subTest(verbosity=verbosity): + output, stdout = self.run_suite(verbosity=verbosity) + self.assertNotIn('Trying:', output) + self.assertNotIn('Expecting:', output) + self.assertEqual(stdout, '') + + def test_verbose(self): + output, stdout = self.run_suite(verbosity=3) + self.assertIn('Trying:\n 2+2\n', output) + self.assertIn('Expecting:\n 4\n', output) + self.assertIn('\nok\n', output) + # Reported to the stream of the test runner, not to the stdout. + self.assertEqual(stdout, '') + + def test_verbose_buffered(self): + # result.buffer replaces sys.stdout, which would swallow the examples. + output, stdout = self.run_suite(verbosity=3, buffer=True) + self.assertIn('Trying:\n 2+2\n', output) + self.assertEqual(stdout, '') + + def test_verbose_failure_not_duplicated(self): + module = 'test.test_doctest.sample_doctest_errors' + quiet, _ = self.run_suite(module, verbosity=2) + verbose, _ = self.run_suite(module, verbosity=3) + self.assertIn('Trying:', verbose) + self.assertNotIn('Trying:', quiet) + # Reporting the examples does not report the failures once more. + self.assertEqual(verbose.count('Failed example:'), + quiet.count('Failed example:')) + self.assertGreater(quiet.count('Failed example:'), 0) + + def test_plain_result(self): + # A result which is not from a text test runner has no stream. + suite = doctest.DocTestSuite('test.test_doctest.sample_doctest') + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + suite.run(unittest.TestResult()) + self.assertEqual(stdout.getvalue(), '') + + class TestDocTestFinder(unittest.TestCase): def test_issue35753(self): diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index f4baa965b3f632..c966f8659e2abb 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -2184,6 +2184,18 @@ def load_tests(loader, tests, pattern): failed=[testname], parallel=True, stats=TestStats(1, 2, 1)) + # A single -v reports the test names, not the examples. + self.assertNotIn('Trying:', output) + + # -vv reports every example, without changing what is run. + output = self.run_tests("--fail-env-changed", "-vv", "-j1", testname, + exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, [testname], + failed=[testname], + parallel=True, + stats=TestStats(1, 2, 1)) + self.assertIn('Trying:\n 1 + 1\n', output) + self.assertIn('Expecting:\n 2\n', output) def _check_random_seed(self, run_workers: bool): # gh-109276: When -r/--randomize is used, random.seed() is called diff --git a/Lib/test/test_unittest/test_discovery.py b/Lib/test/test_unittest/test_discovery.py index 38c9779daaf87d..da184bd12be8d1 100644 --- a/Lib/test/test_unittest/test_discovery.py +++ b/Lib/test/test_unittest/test_discovery.py @@ -635,7 +635,7 @@ def test_command_line_handling_discover_by_default_with_options(self): program._do_discovery = args.append program.parseArgs(['something', '-v', '-b', '-v', '-c', '-f']) self.assertEqual(args, [[]]) - self.assertEqual(program.verbosity, 2) + self.assertEqual(program.verbosity, 3) # -v is passed twice self.assertIs(program.buffer, True) self.assertIs(program.catchbreak, True) self.assertIs(program.failfast, True) diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py index 8ed92373e5e984..a9a73f0d057ee8 100644 --- a/Lib/test/test_unittest/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -267,6 +267,32 @@ def testVerbosity(self): program.parseArgs([None, opt]) self.assertEqual(program.verbosity, 2) + # -v can be repeated to ask for more details. + for args, verbosity in ( + (['-vv'], 3), + (['-v', '-v'], 3), + (['--verbose', '--verbose'], 3), + (['-vvv'], 4), + # -q overrides any number of -v. + (['-v', '-q'], 0), + ): + with self.subTest(args=args): + program.verbosity = 1 + program.parseArgs([None, *args]) + self.assertEqual(program.verbosity, verbosity) + + def testVerbosityCountedOnce(self): + # "python -m unittest -v" falls back to test discovery, which parses + # arguments again: -v must not be counted twice. + program = self.program + program.verbosity = 1 + program.parseArgs([None, '-v']) + self.assertEqual(program.verbosity, 2) + + program.verbosity = 1 + program.parseArgs([None, 'discover', '-vv']) + self.assertEqual(program.verbosity, 3) + def testBufferCatchFailfast(self): program = self.program for arg, attr in (('buffer', 'buffer'), ('failfast', 'failfast'), diff --git a/Lib/test/test_unittest/test_result.py b/Lib/test/test_unittest/test_result.py index 3f44e617303f81..cb591c23fc964f 100644 --- a/Lib/test/test_unittest/test_result.py +++ b/Lib/test/test_unittest/test_result.py @@ -54,6 +54,13 @@ def test_init(self): self.assertEqual(result.shouldStop, False) self.assertIsNone(result._stdout_buffer) self.assertIsNone(result._stderr_buffer) + self.assertEqual(result.verbosity, 1) + + def test_init_verbosity(self): + for verbosity in range(4): + with self.subTest(verbosity=verbosity): + result = unittest.TestResult(None, None, verbosity) + self.assertEqual(result.verbosity, verbosity) # "This method can be called to signal that the set of tests being # run should be aborted by setting the TestResult's shouldStop diff --git a/Lib/test/test_unittest/test_runner.py b/Lib/test/test_unittest/test_runner.py index a47e2ebb59da02..195d5e92803092 100644 --- a/Lib/test/test_unittest/test_runner.py +++ b/Lib/test/test_unittest/test_runner.py @@ -1363,6 +1363,35 @@ def MockResultClass(*args): expectedresult = (runner.stream, DESCRIPTIONS, VERBOSITY) self.assertEqual(runner._makeResult(), expectedresult) + def test_verbosity_set_on_result(self): + class Suite: + def __call__(self, result): + pass + + for verbosity in range(4): + with self.subTest(verbosity=verbosity): + runner = unittest.TextTestRunner(io.StringIO(), + verbosity=verbosity) + result = runner.run(Suite()) + self.assertEqual(result.verbosity, verbosity) + + def test_verbosity_set_on_filtering_result(self): + # A result class is free to filter the verbosity which its + # constructor gets, as test.libregrtest does. + class FilteringResult(unittest.TextTestResult): + def __init__(self, stream, descriptions, verbosity): + super().__init__(stream, descriptions, + 2 if verbosity else 0) + + class Suite: + def __call__(self, result): + pass + + runner = unittest.TextTestRunner(io.StringIO(), verbosity=3, + resultclass=FilteringResult) + result = runner.run(Suite()) + self.assertEqual(result.verbosity, 3) + @support.force_not_colorized @support.requires_subprocess() def test_warnings(self): diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index 6eeebf9657a3c7..850c825a736657 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -161,9 +161,12 @@ def _initArgParsers(self): def _getParentArgParser(self): parser = argparse.ArgumentParser(add_help=False) + # Counted, not a constant: the namespace is the TestProgram, whose + # verbosity is already 1, so -v still gives 2 and -vv gives 3. parser.add_argument('-v', '--verbose', dest='verbosity', - action='store_const', const=2, - help='Verbose output') + action='count', default=1, + help='Verbose output, twice to also report ' + 'the examples of a doctest') parser.add_argument('-q', '--quiet', dest='verbosity', action='store_const', const=0, help='Quiet output') diff --git a/Lib/unittest/result.py b/Lib/unittest/result.py index b8ea396db6772e..a787686a5bea4c 100644 --- a/Lib/unittest/result.py +++ b/Lib/unittest/result.py @@ -37,6 +37,9 @@ class TestResult(object): _moduleSetUpFailed = False def __init__(self, stream=None, descriptions=None, verbosity=None): self.failfast = False + # How much the test runner reports: 0 -- quiet, 1 -- progress dots, + # 2 -- test names, 3 -- also the examples of a doctest. + self.verbosity = 1 if verbosity is None else verbosity self.failures = [] self.errors = [] self.testsRun = 0 diff --git a/Lib/unittest/runner.py b/Lib/unittest/runner.py index 893fcba968c3ef..f19d7b6e446d2a 100644 --- a/Lib/unittest/runner.py +++ b/Lib/unittest/runner.py @@ -244,6 +244,9 @@ def run(self, test): result.failfast = self.failfast result.buffer = self.buffer result.tb_locals = self.tb_locals + # Not left to _makeResult(): a result class is free to filter the + # verbosity which its constructor gets. + result.verbosity = self.verbosity with warnings.catch_warnings(): if self.warnings: # if self.warnings is set, use it to filter all the warnings diff --git a/Misc/NEWS.d/next/Library/2026-08-08-09-14-22.gh-issue-108885.C4ktfw.rst b/Misc/NEWS.d/next/Library/2026-08-08-09-14-22.gh-issue-108885.C4ktfw.rst new file mode 100644 index 00000000000000..e504f9b7d684f8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-08-09-14-22.gh-issue-108885.C4ktfw.rst @@ -0,0 +1,6 @@ +Doctests run by the :mod:`unittest` test runner now report every example, as +in verbose mode, if the runner reports more than the test names, i.e. its +verbosity is 3 or higher. The :option:`!-v` option of :mod:`unittest` can now +be repeated, so ``python -m unittest -vv`` asks for this. Added also the +:attr:`~unittest.TestResult.verbosity` attribute of +:class:`unittest.TestResult`. diff --git a/Misc/NEWS.d/next/Tests/2026-08-08-09-15-08.gh-issue-108885.Qm3vLp.rst b/Misc/NEWS.d/next/Tests/2026-08-08-09-15-08.gh-issue-108885.Qm3vLp.rst new file mode 100644 index 00000000000000..3cdac196dd35c0 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-08-08-09-15-08.gh-issue-108885.Qm3vLp.rst @@ -0,0 +1,2 @@ +Running the Python test suite with ``-vv`` now reports every example of a +doctest. A single ``-v`` reports the test names, as before. From e48b1eb949daacaa84c87da57732866ab1894a86 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 29 Aug 2026 22:34:05 +0300 Subject: [PATCH 10/13] gh-89760: Fix os.path.realpath() for volume GUID paths on Windows (GH-155396) The \\?\ prefix was stripped from the resolved path if it could not be resolved without the prefix and failed with the same error as the original path. This produced an invalid, seemingly relative path for a junction which points to a volume without a drive letter. The prefix is now only stripped for drive-letter and UNC paths. --- Lib/ntpath.py | 38 +++++++++++-------- Lib/test/test_ntpath.py | 27 +++++++++++++ ...26-08-09-00-30-00.gh-issue-89760.rpvol.rst | 2 + 3 files changed, 51 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-09-00-30-00.gh-issue-89760.rpvol.rst diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 071941d3676712..e9e9c4708ee1f1 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -677,6 +677,7 @@ def realpath(path, /, *, strict=False): prefix = b'\\\\?\\' unc_prefix = b'\\\\?\\UNC\\' new_unc_prefix = b'\\\\' + colon_sep = b':\\' # bpo-38081: Special case for realpath(b'nul') devnull = b'nul' if normcase(path) == devnull: @@ -685,6 +686,7 @@ def realpath(path, /, *, strict=False): prefix = '\\\\?\\' unc_prefix = '\\\\?\\UNC\\' new_unc_prefix = '\\\\' + colon_sep = ':\\' # bpo-38081: Special case for realpath('nul') devnull = 'nul' if normcase(path) == devnull: @@ -729,25 +731,29 @@ def realpath(path, /, *, strict=False): # strip off that prefix unless it was already provided on the original # path. if not had_prefix and path.startswith(prefix): - # For UNC paths, the prefix will actually be \\?\UNC\ - # Handle that case as well. + # For UNC drives, the path starts with \\?\UNC\. if path.startswith(unc_prefix): spath = new_unc_prefix + path[len(unc_prefix):] - else: + # For drive-letter drives, the path starts with \\?\:\. + elif path.startswith(colon_sep, len(prefix) + 1): spath = path[len(prefix):] - # Ensure that the non-prefixed path resolves to the same path - try: - if _getfinalpathname(spath) == path: - path = spath - except ValueError: - # Unexpected, as an invalid path should not have gained a prefix - # at any point, but we ignore this error just in case. - pass - except OSError as ex: - # If the path does not exist and originally did not exist, then - # strip the prefix anyway. - if ex.winerror == initial_winerror: - path = spath + # For all others, e.g. volume GUID paths, it cannot be stripped. + else: + spath = None + if spath is not None: + # Ensure that the non-prefixed path resolves to the same path + try: + if _getfinalpathname(spath) == path: + path = spath + except ValueError: + # Unexpected, as an invalid path should not have gained a + # prefix at any point, but we ignore this error just in case. + pass + except OSError as ex: + # If the path does not exist and originally did not exist, + # then strip the prefix anyway. + if ex.winerror == initial_winerror: + path = spath return path diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index fcf98ad88a0f95..a4de2c78f0bf0c 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -1564,6 +1564,33 @@ def test_realpath_drive_relative(self): drive + ':\\spam') self.assertEqual(ntpath.realpath(drive + ':'), drive + ':\\') + @unittest.skipIf(sys.platform != 'win32', "Can only test junctions with creation on win32.") + def test_realpath_volume_guid_path(self): + # gh-89760: the \\?\ prefix cannot be stripped from a volume GUID path. + # Find a volume which is not mounted as a drive. + for volume in os.listvolumes(): + if not os.listmounts(volume): + break + else: + raise unittest.SkipTest('no volume without a mount point') + + with os_helper.temp_dir() as d: + with os_helper.change_cwd(d): + # _winapi.CreateJunction() adds the \\??\\ prefix to a path + # which already has a prefix. + try: + subprocess.run(['cmd', '/c', 'mklink', '/j', + 'testjunc', volume], + check=True, capture_output=True) + except (OSError, subprocess.CalledProcessError): + raise unittest.SkipTest('creating the test junction failed') + + for path in 'testjunc', 'testjunc/spam', 'testjunc/spam/eggs': + with self.subTest(path=path): + realpath = ntpath.realpath(path) + self.assertStartsWith(realpath, '\\\\?\\Volume{') + self.assertTrue(ntpath.isabs(realpath), realpath) + def test_isfile_invalid_paths(self): isfile = ntpath.isfile self.assertIs(isfile('/tmp\udfffabcds'), False) diff --git a/Misc/NEWS.d/next/Library/2026-08-09-00-30-00.gh-issue-89760.rpvol.rst b/Misc/NEWS.d/next/Library/2026-08-09-00-30-00.gh-issue-89760.rpvol.rst new file mode 100644 index 00000000000000..9218ecfe9ce62e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-09-00-30-00.gh-issue-89760.rpvol.rst @@ -0,0 +1,2 @@ +Fix :func:`os.path.realpath` on Windows: the ``\\?\`` prefix is no longer +stripped from a volume GUID path, which made the result invalid. From abf7a184bd94da4d1a6b54503a27ede1957b48b7 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 29 Aug 2026 22:41:39 +0300 Subject: [PATCH 11/13] gh-69370: Fix "python -m inspect --details" for unencodable paths (GH-155246) It failed with UnicodeEncodeError if the path of the module contains characters unencodable in the encoding of sys.stdout, e.g. undecodable bytes of a file name. Such characters are now escaped with backslashes, unless stdout uses an error handler which can handle them. Co-Authored-By: Claude Opus 5 (1M context) --- Lib/inspect.py | 4 ++++ Lib/test/test_inspect/test_inspect.py | 21 ++++++++++++++++++- ...-08-05-17-30-00.gh-issue-69370.inspCLI.rst | 4 ++++ 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-05-17-30-00.gh-issue-69370.inspCLI.rst diff --git a/Lib/inspect.py b/Lib/inspect.py index 3f8991c79652d3..c52469e63861a2 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -3464,6 +3464,10 @@ def _main(): import argparse import importlib + # The printed text can contain characters unencodable in the encoding + # of stdout, e.g. undecodable bytes of a file name. + sys.stdout.reconfigure(errors='backslashreplace') + parser = argparse.ArgumentParser() parser.add_argument( 'object', diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index c68c643cb97fb4..8930f6343ac299 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -40,7 +40,7 @@ from test.support import MISSING_C_DOCSTRINGS, ALWAYS_EQ from test.support import run_no_yield_async_fn, EqualToForwardRef from test.support.import_helper import DirsOnSysPath, ready_to_import -from test.support.os_helper import TESTFN, temp_cwd +from test.support.os_helper import TESTFN, TESTFN_UNDECODABLE, temp_cwd from test.support.script_helper import assert_python_ok, assert_python_failure, kill_python from test.support import has_subprocess_support from test import support @@ -6619,6 +6619,25 @@ def test_error_data(self): lines = err.decode().splitlines() self.assertEqual(lines, [self.NO_SOURCE_TARGET_ERROR]) + @unittest.skipUnless(TESTFN_UNDECODABLE, + 'requires undecodable file names') + def test_details_undecodable_path(self): + # gh-69370: the path of the module is not encodable in the encoding + # of stdout. + with temp_cwd() as test_dir: + subdir = os.path.join(os.fsencode(test_dir), TESTFN_UNDECODABLE) + try: + os.mkdir(subdir) + except OSError: + self.skipTest('undecodable paths are not supported') + with open(os.path.join(subdir, b'undecodable_mod.py'), 'w') as f: + f.write('"""Module docstring."""\n') + rc, out, err = assert_python_ok('-X', 'utf8=0', '-m', 'inspect', + '--details', 'undecodable_mod', + PYTHONPATH=os.fsdecode(subdir)) + self.assertIn(b'Target: undecodable_mod', out) + self.assertEqual(err, b'') + def test_details_option_with_package(self): module_name = 'unittest' module = importlib.import_module(module_name) diff --git a/Misc/NEWS.d/next/Library/2026-08-05-17-30-00.gh-issue-69370.inspCLI.rst b/Misc/NEWS.d/next/Library/2026-08-05-17-30-00.gh-issue-69370.inspCLI.rst new file mode 100644 index 00000000000000..27c97f9bdf07b4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-05-17-30-00.gh-issue-69370.inspCLI.rst @@ -0,0 +1,4 @@ +:program:`python -m inspect --details` no longer fails with +:exc:`UnicodeEncodeError` if the path of the module contains characters +unencodable in the encoding of :data:`sys.stdout`. Such characters are now +escaped with backslashes. From c8ca336e6b321a4ff680ea426cc8a84bfb599bd2 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 29 Aug 2026 22:59:43 +0300 Subject: [PATCH 12/13] gh-69371: Fix pydoc for modules whose path contains undecodable bytes (GH-155239) "python -m pydoc -w" and the pydoc HTTP server failed with UnicodeEncodeError. The file URL is now created with urllib.request.pathname2url(), which percent-encodes the path using the filesystem encoding, and characters unencodable in the generated UTF-8 page are escaped with backslashes. Co-authored-by: Claude Opus 5 (1M context) --- Lib/pydoc.py | 9 +++--- Lib/test/test_pydoc/test_pydoc.py | 28 ++++++++++++++++++- ...6-08-05-16-40-00.gh-issue-69371.pyDocU.rst | 5 ++++ 3 files changed, 37 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-05-16-40-00.gh-issue-69371.pyDocU.rst diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 3cba08b83af681..6f3dab24d5bda3 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -71,7 +71,6 @@ class or function within a module or module in a package. If the import textwrap import time import tokenize -import urllib.parse import warnings from annotationlib import Format from collections import deque @@ -795,7 +794,8 @@ def docmodule(self, object, name=None, mod=None, *ignored): head = linkedname try: path = inspect.getabsfile(object) - url = urllib.parse.quote(path) + import urllib.request + url = urllib.request.pathname2url(path) filelink = self.filelink(url, path) except TypeError: filelink = '(built-in)' @@ -1790,7 +1790,8 @@ def writedoc(thing, forceload=0): """Write HTML documentation to a file in the current directory.""" object, name = resolve(thing, forceload) page = html.page(describe(object), html.document(object, name)) - with open(name + '.html', 'w', encoding='utf-8') as file: + with open(name + '.html', 'w', encoding='utf-8', + errors='backslashreplace') as file: file.write(page) print('wrote', name + '.html') @@ -2387,7 +2388,7 @@ def do_GET(self): self.send_header('Content-Type', '%s; charset=UTF-8' % content_type) self.end_headers() self.wfile.write(self.urlhandler( - self.path, content_type).encode('utf-8')) + self.path, content_type).encode('utf-8', 'backslashreplace')) def log_message(self, *args): # Don't log messages. diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 34d30f54e9c9c7..006fbc8b6968f3 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -19,6 +19,7 @@ import unittest import unittest.mock import urllib.parse +import urllib.request import xml.etree import xml.etree.ElementTree import textwrap @@ -429,7 +430,7 @@ def test_html_doc(self): expected_lines = [line.strip() for line in expected_lines if line] self.assertEqual(text_lines, expected_lines) mod_file = inspect.getabsfile(pydoc_mod) - mod_url = urllib.parse.quote(mod_file) + mod_url = urllib.request.pathname2url(mod_file) self.assertIn(mod_url, result) self.assertIn(mod_file, result) self.assertIn(doc_loc, result) @@ -1024,6 +1025,31 @@ def test_synopsis_sourceless_empty_doc(self): synopsis_cached = pydoc.synopsis(cached_path, {}) self.assertIsNone(synopsis_cached) + @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, + 'requires undecodable file names') + def test_html_doc_undecodable_path(self): + # gh-69371: the path of the module is not encodable in UTF-8. + with os_helper.temp_cwd() as test_dir: + subdir = os.path.join(os.fsencode(test_dir), + os_helper.TESTFN_UNDECODABLE) + try: + os.mkdir(subdir) + except OSError: + self.skipTest('undecodable paths are not supported') + with open(os.path.join(subdir, b'undecodable_mod.py'), 'w') as f: + f.write('"""Module docstring."""\n') + with import_helper.DirsOnSysPath(os.fsdecode(subdir)): + mod = import_helper.import_fresh_module('undecodable_mod') + doc = pydoc.HTMLDoc().docmodule(mod) + with captured_stdout(): + pydoc.writedoc(mod) + # The link contains the percent-encoded path... + path = inspect.getabsfile(mod) + self.assertIn(urllib.request.pathname2url(path), doc) + # ...and the page can be written and served as UTF-8. + with open('undecodable_mod.html', encoding='utf-8') as f: + self.assertIn('undecodable_mod', f.read()) + def test_splitdoc_with_description(self): example_string = "I Am A Doc\n\n\nHere is my description" self.assertEqual(pydoc.splitdoc(example_string), diff --git a/Misc/NEWS.d/next/Library/2026-08-05-16-40-00.gh-issue-69371.pyDocU.rst b/Misc/NEWS.d/next/Library/2026-08-05-16-40-00.gh-issue-69371.pyDocU.rst new file mode 100644 index 00000000000000..4d8f866366f5be --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-05-16-40-00.gh-issue-69371.pyDocU.rst @@ -0,0 +1,5 @@ +Fix :mod:`pydoc` for modules whose path contains undecodable bytes. +:func:`!pydoc.writedoc` and the pydoc HTTP server no longer fail with +:exc:`UnicodeEncodeError`: the file URL is now percent-encoded using the +filesystem encoding, and characters unencodable in the generated HTML page +are escaped with backslashes. From 41b3f0af266b408438cab36625c6ad71040d38e5 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Sat, 29 Aug 2026 22:40:37 +0100 Subject: [PATCH 13/13] gh-156466: fix cleanup on error after Enter in codegen_enter_scope (#156467) --- Python/codegen.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Python/codegen.c b/Python/codegen.c index 875c963a6078c6..79b84f13e629c7 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -700,13 +700,13 @@ codegen_enter_scope(compiler *c, identifier name, int scope_type, * of 0. This is because RETURN_GENERATOR pushes the generator before returning. */ location loc = LOCATION(lineno, lineno, -1, -1); - ADDOP(c, loc, RETURN_GENERATOR); - ADDOP(c, loc, POP_TOP); + ADDOP_IN_SCOPE(c, loc, RETURN_GENERATOR); + ADDOP_IN_SCOPE(c, loc, POP_TOP); } - ADDOP_I(c, loc, RESUME, RESUME_AT_FUNC_START); + ADDOP_I_IN_SCOPE(c, loc, RESUME, RESUME_AT_FUNC_START); if (scope_type == COMPILE_SCOPE_MODULE) { - ADDOP(c, loc, ANNOTATIONS_PLACEHOLDER); + ADDOP_IN_SCOPE(c, loc, ANNOTATIONS_PLACEHOLDER); } return SUCCESS; }