Merge in the main branch

Author: encukou

Doc/howto/abi3t-migration.rst

@@ -217,9 +217,9 @@ Module export hook
 
 Unless you've done this step already, your extension module defines a
 :ref:`module initialization function <extension-pyinit>`
-named :samp:`PyInit_{<module_name>}`.
+named :samp:`PyInit_{<modname>}` (where ``modname`` is the name of your module).
 You will need to port it to a :ref:`module export hook <extension-export-hook>`,
-:samp:`PyModExport_{<module name>}`, a feature added in CPython 3.15 in
+:samp:`PyModExport_{<modname>}`, a feature added in CPython 3.15 in
 :pep:`793`.
 
 Your existing init function should look like this (with your own names
@@ -297,6 +297,34 @@ pointer to static data.
 If you cannot avoid additional code, refer to the
 :ref:`caveats in PyModExport documentation <pymodexport-api-caveats>`.
 
+.. note::
+
+   When building for Windows using the Setuptools_ build tool,
+   removing the :samp:`PyInit_{<modname>}` function may result in the linker error
+   :samp:`LINK : error LNK2001: unresolved external symbol PyInit_{<modname>}`.
+   This is caused by Setuptools passing an ``/EXPORT`` linker flag, which
+   is redundant since Python 3.15 (see :gh:`141671`).
+   A workaround is to add a dummy :samp:`PyInit_{<modname>}` function
+   to your code.
+   Python 3.15+ will never call this function if
+   :samp:`PyModExport_{<modname>}` is present, so it can always fail:
+
+   .. code-block:: c
+
+      // Workaround for https://github.com/pypa/distutils/issues/387
+      PyMODINIT_FUNC
+      PyInit_<modname>(void)
+      {
+          PyErr_SetString(PyExc_SystemError,
+                          "PyInit_* called for module with PyModExport_*");
+          return NULL;
+      }
+
+   (This issue is present in Setuptools 84.0.0; it might be fixed in newer
+   versions.)
+
+.. _Setuptools: https://setuptools.pypa.io/
+
 
 Existing slots
 --------------

Doc/library/codecs.rst

@@ -1395,15 +1395,6 @@ encodings.
 |                    |         | :mod:`encodings.idna`.    |
 |                    |         | Only ``errors='strict'``  |
 |                    |         | is supported.             |
-|                    |         |                           |
-|                    |         | .. warning::              |
-|                    |         |                           |
-|                    |         |    This codec builds on   |
-|                    |         |    ``punycode``, whose    |
-|                    |         |    algorithms scale       |
-|                    |         |    poorly, so limit the   |
-|                    |         |    length of untrusted    |
-|                    |         |    input.                 |
 +--------------------+---------+---------------------------+
 | mbcs               | ansi,   | Windows only: Encode the  |
 |                    | dbcs    | operand according to the  |
@@ -1655,11 +1646,6 @@ Applications) and :rfc:`3492` (Nameprep: A Stringprep Profile for
 Internationalized Domain Names (IDN)). It builds upon the ``punycode`` encoding
 and :mod:`stringprep`.
 
-.. warning::
-
-   This module builds on ``punycode``, whose algorithms scale poorly, so limit
-   the length of untrusted input.
-
 If you need the IDNA 2008 standard from :rfc:`5891` and :rfc:`5895`, use the
 third-party :pypi:`idna` module.
 
@@ -1697,11 +1683,31 @@ international domain names, and to unify similar characters. The nameprep
 functions can be used directly if desired.
 
 
-.. function:: nameprep(label)
+.. function:: nameprep(label, *, limit=None)
 
    Return the nameprepped version of *label*. The implementation currently assumes
    query strings, so ``AllowUnassigned`` is true.
 
+   Raise :exc:`UnicodeEncodeError` if the nameprep algorithm emits an error.
+
+   If the *limit* argument is given, it should be set to the maximum size
+   of an encoded A-label (that is, 63 for IDNA).
+   :func:`!nameprep` will raise :exc:`UnicodeEncodeError` if the label is
+   **much** larger than *limit*.
+   Note that this is only a rough check meant to skip expensive processing
+   of extremely large input; the caller should check any exact
+   limits separately.
+
+   .. warning::
+
+      For backwards compatibility, label size is unlimited by default.
+      This may cause issues when processing the result with the
+      ``punycode`` encoding, whose algorithms scale poorly.
+
+   .. versionchanged:: next
+
+      Added the *limit* parameter.
+
 
 .. function:: ToASCII(label)
 

Doc/whatsnew/3.16.rst

@@ -397,6 +397,10 @@ encodings
   used for international IMAP4 mailbox names (:rfc:`3501`).
   (Contributed by Serhiy Storchaka in :gh:`66788`.)
 
+* :func:`encodings.idna.nameprep` now takes a *limit* argument that allows
+  rejecting extremely large input early.
+  (Contributed by Petr Viktorin in :gh:`157675`.)
+
 
 gzip
 ----

Include/internal/pycore_import.h

@@ -89,6 +89,8 @@ extern void _PyImport_ClearModulesByIndex(PyInterpreterState *interp);
 extern PyObject * _PyImport_InitLazyModules(
     PyInterpreterState *interp);
 extern void _PyImport_ClearLazyModules(PyInterpreterState *interp);
+extern int _PyImport_DiscardLazyModule(
+    PyInterpreterState *interp, PyObject *name);
 
 extern int _PyImport_InitDefaultImportFunc(PyInterpreterState *interp);
 extern int _PyImport_IsDefaultImportFunc(

Include/internal/pycore_pystate.h

@@ -335,14 +335,16 @@ static uintptr_t return_pointer_as_int(char* p) {
 static inline uintptr_t
 _Py_get_machine_stack_pointer(void) {
     uintptr_t result;
-#if defined(_M_ARM64)
+#if _Py__has_builtin(__builtin_stack_address)
+    result = (uintptr_t)__builtin_stack_address();
+#elif defined(_MSC_VER) && defined(_M_ARM64)
     result = __getReg(31);
-#elif defined(_M_X64) || defined(_M_IX86)
+#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86))
     result = (uintptr_t)_AddressOfReturnAddress();
 #elif defined(__aarch64__)
     __asm__ ("mov %0, sp" : "=r" (result));
 #elif defined(__x86_64__)
-    __asm__("{movq %%rsp, %0" : "=r" (result));
+    __asm__ ("{movq %%rsp, %0|mov %0, rsp}" : "=r" (result));
 #else
     char here;
     result = (uintptr_t)&here;

Lib/asyncio/tasks.py

@@ -938,25 +938,6 @@ def _done_callback(fut, cur_task=cur_task):
     return outer
 
 
-def _log_on_exception(fut):
-    if fut.cancelled():
-        return
-
-    exc = fut.exception()
-    if exc is None:
-        return
-
-    context = {
-        'message':
-        f'{exc.__class__.__name__} exception in shielded future',
-        'exception': exc,
-        'future': fut,
-    }
-    if fut._source_traceback:
-        context['source_traceback'] = fut._source_traceback
-    fut._loop.call_exception_handler(context)
-
-
 def shield(arg):
     """Wait for a future, shielding it from cancellation.
 
@@ -1021,9 +1002,6 @@ def _inner_done_callback(inner):
     def _outer_done_callback(outer):
         if not inner.done():
             inner.remove_done_callback(_inner_done_callback)
-            # Keep only one callback to log on cancel
-            inner.remove_done_callback(_log_on_exception)
-            inner.add_done_callback(_log_on_exception)
             if cur_task is not None:
                 inner.remove_done_callback(_clear_awaited_by_callback)
                 futures.future_discard_from_awaited_by(inner, cur_task)

Lib/encodings/idna.py

@@ -1,5 +1,6 @@
 # This module implements the RFCs 3490 (IDNA) and 3491 (Nameprep)
 
+import sys
 import stringprep, re, codecs
 from unicodedata import ucd_3_2_0 as unicodedata
 
@@ -11,14 +12,36 @@
 sace_prefix = "xn--"
 
 # This assumes query strings, so AllowUnassigned is true
-def nameprep(label):  # type: (str) -> str
+def nameprep(label, *, limit=None):  # type: (str) -> str
+    if limit is None:
+        limit = sys.maxsize
+    else:
+        # Protection from gh-98433 and gh-157675 (passing unbounded input to
+        # the quadratic-complexity punycode algorithm).
+        # While the "map" step can remove characters, later steps (in ToASCII
+        # and FromASCII) will not shorten the result *drastically*.
+        # (NFKC normalization can compress e.g. '\u03c9\u0314\u0300\u0345'
+        # to '\u1fa3' -- a 4-fold reduction. Non-ASCII labels then get
+        # longer via prefixing & punycode).
+        # We bail if the number of non-ignored input characters exceeds 8 times
+        # the limit, which gives ample room for future Unicode versions to
+        # include long normalizations, while still preventing us from wasting
+        # time decoding a big thing  that'll just hit the actual <= 63 limit in
+        # ToASCII.
+        limit *= 8
+
     # Map
     newlabel = []
     for c in label:
         if stringprep.in_table_b1(c):
             # Map to nothing
             continue
         newlabel.append(stringprep.map_table_b2(c))
+
+        if len(newlabel) > limit:
+            raise UnicodeEncodeError("idna", label, 0, len(label),
+                                     "label way too long")
+
     label = "".join(newlabel)
 
     # Normalize
@@ -80,7 +103,7 @@ def ToASCII(label):  # type: (str) -> bytes
             raise UnicodeEncodeError("idna", label, 0, len(label), "label too long")
 
     # Step 2: nameprep
-    label = nameprep(label)
+    label = nameprep(label, limit=63)
 
     # Step 3: UseSTD3ASCIIRules is false
     # Step 4: try ASCII
@@ -115,18 +138,6 @@ def ToASCII(label):  # type: (str) -> bytes
     raise UnicodeEncodeError("idna", label, 0, len(label), "label too long")
 
 def ToUnicode(label):
-    if len(label) > 1024:
-        # Protection from https://github.com/python/cpython/issues/98433.
-        # https://datatracker.ietf.org/doc/html/rfc5894#section-6
-        # doesn't specify a label size limit prior to NAMEPREP. But having
-        # one makes practical sense.
-        # This leaves ample room for nameprep() to remove Nothing characters
-        # per https://www.rfc-editor.org/rfc/rfc3454#section-3.1 while still
-        # preventing us from wasting time decoding a big thing that'll just
-        # hit the actual <= 63 length limit in Step 6.
-        if isinstance(label, str):
-            label = label.encode("utf-8", errors="backslashreplace")
-        raise UnicodeDecodeError("idna", label, 0, len(label), "label way too long")
     # Step 1: Check for ASCII
     if isinstance(label, bytes):
         pure_ascii = True
@@ -139,7 +150,7 @@ def ToUnicode(label):
     if not pure_ascii:
         assert isinstance(label, str)
         # Step 2: Perform nameprep
-        label = nameprep(label)
+        label = nameprep(label, limit=63)
         # It doesn't say this, but apparently, it should be ASCII now
         try:
             label = label.encode("ascii")
@@ -151,6 +162,11 @@ def ToUnicode(label):
     if not label.lower().startswith(ace_prefix):
         return str(label, "ascii")
 
+    # Below in steps 6-7, `label` must match the result of `ToASCII`, so it's
+    # limited to 63 chars. Check before the expensive punycode decode.
+    if len(label) >= 64:
+        raise UnicodeDecodeError("idna", label, 0, len(label), "label too long")
+
     # Step 4: Remove ACE prefix
     label1 = label[len(ace_prefix):]
 

Lib/test/support/__init__.py

@@ -3525,17 +3525,12 @@ def check_immutable_type(testcase, type):
 
 def built_with_c_assertions():
     """Check if Python was built with C assertions (assert())."""
-
-    if MS_WINDOWS:
-        # On Windows, rely on the Py_DEBUG macro to check for assertions
+    try:
+        import _testlimitedcapi
+    except ImportError:
         return Py_DEBUG
-
-    # Check if the NDEBUG macro is defined in C compiler flags
-    PY_CFLAGS = (sysconfig.get_config_var('PY_CFLAGS') or '')
-    if '-DNDEBUG' in PY_CFLAGS:
-        return False
-
-    return True
+    else:
+        return bool(_testlimitedcapi._py_getbuiltwithassert())
 
 
 def inject_memory_error(start=0, stop=0):

Lib/test/test_array.py

@@ -12,6 +12,7 @@
 import weakref
 import pickle
 import operator
+import re
 import struct
 import sys
 
@@ -110,6 +111,25 @@ def test_typecodes(self):
             self.assertIsInstance(typecode, str)
             self.assertGreaterEqual(len(typecode), 1)
 
+    @support.cpython_only
+    @support.requires_docstrings
+    def test_typecodes_documented(self):
+        # The type code table in the array.array docstring must list every
+        # supported type code, and nothing else, and the minimum size it
+        # gives for each of them must be one the implementation meets.
+        row = re.compile(r"^ {4}'(\w+)' +\S.*? +(\d+)(?: \(see note\))?$")
+        documented = {}
+        for line in array.array.__doc__.splitlines():
+            match = row.match(line)
+            if match is not None:
+                documented[match.group(1)] = int(match.group(2))
+
+        self.assertEqual(sorted(documented), sorted(array.typecodes))
+        for typecode, minimum_size in documented.items():
+            with self.subTest(typecode=typecode):
+                self.assertGreaterEqual(array.array(typecode).itemsize,
+                                        minimum_size)
+
 
 # Machine format codes.
 #