Merge remote-tracking branch 'upstream/main' into tachyon-land-158581

Author: pablogsal

.github/workflows/build.yml

@@ -547,6 +547,9 @@ jobs:
         - check-name: Undefined behavior
           sanitizer: UBSan
           free-threading: false
+        - check-name: Memory
+          sanitizer: MSan
+          free-threading: false
     uses: ./.github/workflows/reusable-san.yml
     with:
       sanitizer: ${{ matrix.sanitizer }}

.github/workflows/reusable-san.yml

@@ -60,7 +60,7 @@ jobs:
             || ''
           }}
     - name: UBSan option setup
-      if: inputs.sanitizer != 'TSan'
+      if: inputs.sanitizer == 'UBSan'
       run: >-
         echo
         "UBSAN_OPTIONS=${SAN_LOG_OPTION}
@@ -69,6 +69,20 @@ jobs:
         >> "$GITHUB_ENV"
       env:
         SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log
+    - name: MSan option setup
+      if: inputs.sanitizer == 'MSan'
+      run: |
+        echo "MSAN_OPTIONS=${SAN_LOG_OPTION} allocator_may_return_null=1 handle_segv=0" >> "$GITHUB_ENV"
+        # MSan reports false positives for memory initialized by libraries
+        # that are not built with MSan, so disable modules that use them.
+        # _remote_debugging links to libzstd directly, but we unpoision the memory.
+        {
+          echo '*disabled*'
+          echo '_bz2 _ctypes _curses _curses_panel _dbm _decimal _gdbm _hashlib'
+          echo '_lzma _sqlite3 _ssl _tkinter _uuid _zstd readline zlib'
+        } > Modules/Setup.local
+      env:
+        SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log
     - name: Add ccache to PATH
       run: |
         echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
@@ -93,6 +107,8 @@ jobs:
     # gh-157958: -O2 instead of the pydebug default -Og to avoid a clang 21
     # compile-time blowup on some interpreter files.
     # (https://github.com/llvm/llvm-project/issues/179695)
+    # MSan uses --with-assertions instead of --with-pydebug because its
+    # hooks on the Python memory allocators hide uninitialized reads.
     - name: Configure CPython
       run: >-
         ./configure
@@ -101,9 +117,11 @@ jobs:
         ${{
           inputs.sanitizer == 'TSan'
           && '--with-thread-sanitizer'
+          || inputs.sanitizer == 'MSan'
+          && '--with-memory-sanitizer'
           || '--with-undefined-behavior-sanitizer --with-strict-overflow'
         }}
-        --with-pydebug
+        ${{ inputs.sanitizer == 'MSan' && '--with-assertions' || '--with-pydebug' }}
         ${{ inputs.sanitizer == 'TSan' && '--with-openssl="$OPENSSL_DIR" --with-openssl-rpath=auto' || '' }}
         ${{ inputs.free-threading && '--disable-gil' || '' }}
     - name: Build CPython

Doc/c-api/exceptions.rst

@@ -1034,6 +1034,10 @@ because the :ref:`call protocol <call>` takes care of recursion handling.
    case, a :exc:`RecursionError` is set and a nonzero value is returned.
    Otherwise, zero is returned.
 
+   The check is based on the remaining C stack space of the current thread,
+   not on a count of calls, so it is unaffected by
+   :c:func:`Py_SetRecursionLimit` and :func:`sys.setrecursionlimit`.
+
    *where* should be a UTF-8 encoded string such as ``" in instance check"`` to
    be concatenated to the :exc:`RecursionError` message caused by the recursion
    depth limit.
@@ -1044,6 +1048,10 @@ because the :ref:`call protocol <call>` takes care of recursion handling.
    .. versionchanged:: 3.9
       This function is now also available in the :ref:`limited API <limited-c-api>`.
 
+   .. versionchanged:: 3.14
+      The check is based on the remaining C stack space.  Previously, a
+      separate counter of C-level calls was used.
+
 .. c:function:: void Py_LeaveRecursiveCall(void)
 
    Ends a :c:func:`Py_EnterRecursiveCall`.  Must be called once for each

Doc/c-api/interp-lifecycle.rst

@@ -514,7 +514,7 @@ Process-wide parameters
    Return the version of this Python interpreter.  This is a string that looks
    something like ::
 
-      "3.0a5+ (py3k:63103M, May 12 2008, 00:53:55) \n[GCC 4.2.3]"
+      "3.15.0rc2 (3.15.0~rc2-1.fc44.x86_64, Sep  3 2026, 00:00:00) [GCC 16.2.1 20260819 (Red Hat 16.2.1-2)]"
 
    .. index:: single: version (in module sys)
 

Doc/c-api/sys.rst

@@ -152,28 +152,29 @@ Operating System Utilities
       <c-preinit>` and so that the LC_CTYPE locale is properly configured: see
       the :c:func:`Py_PreInitialize` function.
 
-   Decode a byte string from the :term:`filesystem encoding and error handler`.
-   If the error handler is :ref:`surrogateescape error handler
-   <surrogateescape>`, undecodable bytes are decoded as characters in range
-   U+DC80..U+DCFF; and if a byte sequence can be decoded as a surrogate
-   character, the bytes are escaped using the surrogateescape error handler
-   instead of decoding them.
+   Decode a byte string from the :term:`filesystem encoding <filesystem
+   encoding and error handler>` with the :ref:`surrogateescape error handler
+   <surrogateescape>`.
+
+   Undecodable bytes are decoded as characters in range U+DC80..U+DCFF. If a
+   byte sequence can be decoded as a surrogate character, escape the bytes
+   using the surrogateescape error handler instead of decoding them.
 
    Return a pointer to a newly allocated wide character string, use
    :c:func:`PyMem_RawFree` to free the memory. If size is not ``NULL``, write
    the number of wide characters excluding the null character into ``*size``
 
-   Return ``NULL`` on decoding error or memory allocation error. If *size* is
-   not ``NULL``, ``*size`` is set to ``(size_t)-1`` on memory error or set to
-   ``(size_t)-2`` on decoding error.
+   On memory allocation failure, set *\*size* to ``(size_t)-1`` and return
+   ``NULL``.
+
+   On decode error, set *\*size* to ``(size_t)-2`` and return ``NULL``.
+   Decoding errors should never happen, unless there is a bug in the C
+   library.
 
    The :term:`filesystem encoding and error handler` are selected by
    :c:func:`PyConfig_Read`: see :c:member:`~PyConfig.filesystem_encoding` and
    :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`.
 
-   Decoding errors should never happen, unless there is a bug in the C
-   library.
-
    Use the :c:func:`Py_EncodeLocale` function to encode the character string
    back to a byte string.
 
@@ -195,17 +196,19 @@ Operating System Utilities
 
 .. c:function:: char* Py_EncodeLocale(const wchar_t *text, size_t *error_pos)
 
-   Encode a wide character string to the :term:`filesystem encoding and error
-   handler`. If the error handler is :ref:`surrogateescape error handler
-   <surrogateescape>`, surrogate characters in the range U+DC80..U+DCFF are
-   converted to bytes 0x80..0xFF.
+   Encode a wide character string to the :term:`filesystem encoding <filesystem
+   encoding and error handler>` with the :ref:`surrogateescape error handler
+   <surrogateescape>`. Surrogate characters in the range U+DC80..U+DCFF are
+   encoded to bytes 0x80..0xFF.
 
    Return a pointer to a newly allocated byte string, use :c:func:`PyMem_Free`
-   to free the memory. Return ``NULL`` on encoding error or memory allocation
-   error.
+   to free the memory.
+
+   On memory allocation failure, set *\*error_pos* to ``(size_t)-1`` and return
+   ``NULL``.
 
-   If error_pos is not ``NULL``, ``*error_pos`` is set to ``(size_t)-1`` on
-   success,  or set to the index of the invalid character on encoding error.
+   On encoding error, set *\*error_pos* to the index of the first unencodable
+   character and return ``NULL``.
 
    The :term:`filesystem encoding and error handler` are selected by
    :c:func:`PyConfig_Read`: see :c:member:`~PyConfig.filesystem_encoding` and

Doc/c-api/unicode.rst

@@ -168,8 +168,14 @@ access to internal read-only data of Unicode objects:
    The function performs no checks for any of its requirements,
    and is intended for usage in loops.
 
+   While :class:`str` objects are usually immutable in Python, this special C API allows
+   mutating a fresh :class:`str` object if the string has not been "used" yet.
+
    .. versionadded:: 3.3
 
+   .. soft-deprecated:: next
+      Use the :c:type:`PyUnicodeWriter` API instead.
+
 
 .. c:function:: Py_UCS4 PyUnicode_READ(int kind, void *data, Py_ssize_t index)
 
@@ -407,9 +413,15 @@ APIs:
    using the :c:type:`PyUnicodeWriter` API, or one of the ``PyUnicode_From*``
    functions below.
 
+   While :class:`str` objects are usually immutable in Python, this special C API
+   returns a :class:`str` object that can be mutated, except if *size* is zero, in which
+   case it returns the immutable empty string constant.
 
    .. versionadded:: 3.3
 
+   .. soft-deprecated:: next
+      Use the :c:type:`PyUnicodeWriter` API instead.
+
 
 .. c:function:: PyObject* PyUnicode_FromKindAndData(int kind, const void *buffer, \
                                                     Py_ssize_t size)
@@ -754,11 +766,16 @@ APIs:
    possible.  Returns ``-1`` and sets an exception on error, otherwise returns
    the number of copied characters.
 
-   The string must not have been “used” yet.
+   While :class:`str` objects are usually immutable in Python, this special C API allows
+   mutating a fresh :class:`str` object if the string has not been "used" yet.
+
    See :c:func:`PyUnicode_New` for details.
 
    .. versionadded:: 3.3
 
+   .. soft-deprecated:: next
+      Use the :c:type:`PyUnicodeWriter` API instead.
+
 
 .. c:function:: int PyUnicode_Resize(PyObject **unicode, Py_ssize_t length);
 
@@ -774,6 +791,14 @@ APIs:
    The function doesn't check string content, the result may not be a
    string in canonical representation.
 
+   While :class:`str` objects are usually immutable in Python, this special C API
+   can resize a :class:`str` object in-place if the string has not been "used" yet.
+   It returns a :class:`str` object which can be mutated, except if *size* is zero, in
+   which case it returns the immutable empty string constant.
+
+   .. soft-deprecated:: next
+      Use the :c:type:`PyUnicodeWriter` API instead.
+
 
 .. c:function:: Py_ssize_t PyUnicode_Fill(PyObject *unicode, Py_ssize_t start, \
                         Py_ssize_t length, Py_UCS4 fill_char)
@@ -784,14 +809,19 @@ APIs:
    Fail if *fill_char* is bigger than the string maximum character, or if the
    string has more than 1 reference.
 
-   The string must not have been “used” yet.
-   See :c:func:`PyUnicode_New` for details.
-
    Return the number of written characters, or return ``-1`` and raise an
    exception on error.
 
+   While :class:`str` objects are usually immutable in Python, this special C API allows
+   mutating a fresh :class:`str` object if the string has not been "used" yet.
+
+   See :c:func:`PyUnicode_New` for details.
+
    .. versionadded:: 3.3
 
+   .. soft-deprecated:: next
+      Use the :c:type:`PyUnicodeWriter` API instead.
+
 
 .. c:function:: int PyUnicode_WriteChar(PyObject *unicode, Py_ssize_t index, \
                                         Py_UCS4 character)
@@ -804,11 +834,16 @@ APIs:
    See :c:func:`PyUnicode_WRITE` for a version that skips these checks,
    making them your responsibility.
 
-   The string must not have been “used” yet.
+   While :class:`str` objects are usually immutable in Python, this special C API allows
+   mutating a fresh :class:`str` object if the string has not been "used" yet.
+
    See :c:func:`PyUnicode_New` for details.
 
    .. versionadded:: 3.3
 
+   .. soft-deprecated:: next
+      Use the :c:type:`PyUnicodeWriter` API instead.
+
 
 .. c:function:: Py_UCS4 PyUnicode_ReadChar(PyObject *unicode, Py_ssize_t index)
 

Doc/howto/abi3t-migration.rst

@@ -217,9 +217,9 @@ Module export hook
 
 Unless you've done this step already, your extension module defines a
 :ref:`module initialization function <extension-pyinit>`
-named :samp:`PyInit_{<module_name>}`.
+named :samp:`PyInit_{<modname>}` (where ``modname`` is the name of your module).
 You will need to port it to a :ref:`module export hook <extension-export-hook>`,
-:samp:`PyModExport_{<module name>}`, a feature added in CPython 3.15 in
+:samp:`PyModExport_{<modname>}`, a feature added in CPython 3.15 in
 :pep:`793`.
 
 Your existing init function should look like this (with your own names
@@ -297,6 +297,34 @@ pointer to static data.
 If you cannot avoid additional code, refer to the
 :ref:`caveats in PyModExport documentation <pymodexport-api-caveats>`.
 
+.. note::
+
+   When building for Windows using the Setuptools_ build tool,
+   removing the :samp:`PyInit_{<modname>}` function may result in the linker error
+   :samp:`LINK : error LNK2001: unresolved external symbol PyInit_{<modname>}`.
+   This is caused by Setuptools passing an ``/EXPORT`` linker flag, which
+   is redundant since Python 3.15 (see :gh:`141671`).
+   A workaround is to add a dummy :samp:`PyInit_{<modname>}` function
+   to your code.
+   Python 3.15+ will never call this function if
+   :samp:`PyModExport_{<modname>}` is present, so it can always fail:
+
+   .. code-block:: c
+
+      // Workaround for https://github.com/pypa/distutils/issues/387
+      PyMODINIT_FUNC
+      PyInit_<modname>(void)
+      {
+          PyErr_SetString(PyExc_SystemError,
+                          "PyInit_* called for module with PyModExport_*");
+          return NULL;
+      }
+
+   (This issue is present in Setuptools 84.0.0; it might be fixed in newer
+   versions.)
+
+.. _Setuptools: https://setuptools.pypa.io/
+
 
 Existing slots
 --------------

Doc/library/codecs.rst

@@ -1395,15 +1395,6 @@ encodings.
 |                    |         | :mod:`encodings.idna`.    |
 |                    |         | Only ``errors='strict'``  |
 |                    |         | is supported.             |
-|                    |         |                           |
-|                    |         | .. warning::              |
-|                    |         |                           |
-|                    |         |    This codec builds on   |
-|                    |         |    ``punycode``, whose    |
-|                    |         |    algorithms scale       |
-|                    |         |    poorly, so limit the   |
-|                    |         |    length of untrusted    |
-|                    |         |    input.                 |
 +--------------------+---------+---------------------------+
 | mbcs               | ansi,   | Windows only: Encode the  |
 |                    | dbcs    | operand according to the  |
@@ -1655,11 +1646,6 @@ Applications) and :rfc:`3492` (Nameprep: A Stringprep Profile for
 Internationalized Domain Names (IDN)). It builds upon the ``punycode`` encoding
 and :mod:`stringprep`.
 
-.. warning::
-
-   This module builds on ``punycode``, whose algorithms scale poorly, so limit
-   the length of untrusted input.
-
 If you need the IDNA 2008 standard from :rfc:`5891` and :rfc:`5895`, use the
 third-party :pypi:`idna` module.
 
@@ -1697,11 +1683,31 @@ international domain names, and to unify similar characters. The nameprep
 functions can be used directly if desired.
 
 
-.. function:: nameprep(label)
+.. function:: nameprep(label, *, limit=None)
 
    Return the nameprepped version of *label*. The implementation currently assumes
    query strings, so ``AllowUnassigned`` is true.
 
+   Raise :exc:`UnicodeEncodeError` if the nameprep algorithm emits an error.
+
+   If the *limit* argument is given, it should be set to the maximum size
+   of an encoded A-label (that is, 63 for IDNA).
+   :func:`!nameprep` will raise :exc:`UnicodeEncodeError` if the label is
+   **much** larger than *limit*.
+   Note that this is only a rough check meant to skip expensive processing
+   of extremely large input; the caller should check any exact
+   limits separately.
+
+   .. warning::
+
+      For backwards compatibility, label size is unlimited by default.
+      This may cause issues when processing the result with the
+      ``punycode`` encoding, whose algorithms scale poorly.
+
+   .. versionchanged:: next
+
+      Added the *limit* parameter.
+
 
 .. function:: ToASCII(label)
 

Doc/library/ctypes.rst

@@ -1755,8 +1755,9 @@ These prefabricated library loaders are available:
       :c:expr:`int`, which is of course not always the truth, so you have to assign
       the correct :attr:`!restype` attribute to use these functions.
 
-      Note that if the Python interpreter is statically linked, this will be
-      ``None``, as ``dlopen`` is not possible in this case.
+      .. note::
+
+         If the Python interpreter is statically linked, this may be ``None``.
 
 .. audit-event:: ctypes.dlopen name ctypes.LibraryLoader
 

Doc/using/configure.rst

@@ -1026,6 +1026,10 @@ Debug options
 
    Enable MemorySanitizer allocation error detector, ``msan`` (default is no).
 
+   MSan reports false positives for memory initialized by libraries that are
+   not built with MSan, so either build all dependencies with MSan or disable
+   the extension modules that use them in :file:`Modules/Setup.local`.
+
    .. versionadded:: 3.6
 
 .. option:: --with-undefined-behavior-sanitizer