Merge branch 'main' into dict_delitem_common_optimization

Author: hetaozdh

.github/workflows/build.yml

@@ -547,6 +547,9 @@ jobs:
         - check-name: Undefined behavior
           sanitizer: UBSan
           free-threading: false
+        - check-name: Memory
+          sanitizer: MSan
+          free-threading: false
     uses: ./.github/workflows/reusable-san.yml
     with:
       sanitizer: ${{ matrix.sanitizer }}

.github/workflows/reusable-san.yml

@@ -60,7 +60,7 @@ jobs:
             || ''
           }}
     - name: UBSan option setup
-      if: inputs.sanitizer != 'TSan'
+      if: inputs.sanitizer == 'UBSan'
       run: >-
         echo
         "UBSAN_OPTIONS=${SAN_LOG_OPTION}
@@ -69,6 +69,20 @@ jobs:
         >> "$GITHUB_ENV"
       env:
         SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log
+    - name: MSan option setup
+      if: inputs.sanitizer == 'MSan'
+      run: |
+        echo "MSAN_OPTIONS=${SAN_LOG_OPTION} allocator_may_return_null=1 handle_segv=0" >> "$GITHUB_ENV"
+        # MSan reports false positives for memory initialized by libraries
+        # that are not built with MSan, so disable modules that use them.
+        # _remote_debugging links to libzstd directly, but we unpoision the memory.
+        {
+          echo '*disabled*'
+          echo '_bz2 _ctypes _curses _curses_panel _dbm _decimal _gdbm _hashlib'
+          echo '_lzma _sqlite3 _ssl _tkinter _uuid _zstd readline zlib'
+        } > Modules/Setup.local
+      env:
+        SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log
     - name: Add ccache to PATH
       run: |
         echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
@@ -93,6 +107,8 @@ jobs:
     # gh-157958: -O2 instead of the pydebug default -Og to avoid a clang 21
     # compile-time blowup on some interpreter files.
     # (https://github.com/llvm/llvm-project/issues/179695)
+    # MSan uses --with-assertions instead of --with-pydebug because its
+    # hooks on the Python memory allocators hide uninitialized reads.
     - name: Configure CPython
       run: >-
         ./configure
@@ -101,9 +117,11 @@ jobs:
         ${{
           inputs.sanitizer == 'TSan'
           && '--with-thread-sanitizer'
+          || inputs.sanitizer == 'MSan'
+          && '--with-memory-sanitizer'
           || '--with-undefined-behavior-sanitizer --with-strict-overflow'
         }}
-        --with-pydebug
+        ${{ inputs.sanitizer == 'MSan' && '--with-assertions' || '--with-pydebug' }}
         ${{ inputs.sanitizer == 'TSan' && '--with-openssl="$OPENSSL_DIR" --with-openssl-rpath=auto' || '' }}
         ${{ inputs.free-threading && '--disable-gil' || '' }}
     - name: Build CPython

Doc/c-api/sys.rst

@@ -152,28 +152,29 @@ Operating System Utilities
       <c-preinit>` and so that the LC_CTYPE locale is properly configured: see
       the :c:func:`Py_PreInitialize` function.
 
-   Decode a byte string from the :term:`filesystem encoding and error handler`.
-   If the error handler is :ref:`surrogateescape error handler
-   <surrogateescape>`, undecodable bytes are decoded as characters in range
-   U+DC80..U+DCFF; and if a byte sequence can be decoded as a surrogate
-   character, the bytes are escaped using the surrogateescape error handler
-   instead of decoding them.
+   Decode a byte string from the :term:`filesystem encoding <filesystem
+   encoding and error handler>` with the :ref:`surrogateescape error handler
+   <surrogateescape>`.
+
+   Undecodable bytes are decoded as characters in range U+DC80..U+DCFF. If a
+   byte sequence can be decoded as a surrogate character, escape the bytes
+   using the surrogateescape error handler instead of decoding them.
 
    Return a pointer to a newly allocated wide character string, use
    :c:func:`PyMem_RawFree` to free the memory. If size is not ``NULL``, write
    the number of wide characters excluding the null character into ``*size``
 
-   Return ``NULL`` on decoding error or memory allocation error. If *size* is
-   not ``NULL``, ``*size`` is set to ``(size_t)-1`` on memory error or set to
-   ``(size_t)-2`` on decoding error.
+   On memory allocation failure, set *\*size* to ``(size_t)-1`` and return
+   ``NULL``.
+
+   On decode error, set *\*size* to ``(size_t)-2`` and return ``NULL``.
+   Decoding errors should never happen, unless there is a bug in the C
+   library.
 
    The :term:`filesystem encoding and error handler` are selected by
    :c:func:`PyConfig_Read`: see :c:member:`~PyConfig.filesystem_encoding` and
    :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`.
 
-   Decoding errors should never happen, unless there is a bug in the C
-   library.
-
    Use the :c:func:`Py_EncodeLocale` function to encode the character string
    back to a byte string.
 
@@ -195,17 +196,19 @@ Operating System Utilities
 
 .. c:function:: char* Py_EncodeLocale(const wchar_t *text, size_t *error_pos)
 
-   Encode a wide character string to the :term:`filesystem encoding and error
-   handler`. If the error handler is :ref:`surrogateescape error handler
-   <surrogateescape>`, surrogate characters in the range U+DC80..U+DCFF are
-   converted to bytes 0x80..0xFF.
+   Encode a wide character string to the :term:`filesystem encoding <filesystem
+   encoding and error handler>` with the :ref:`surrogateescape error handler
+   <surrogateescape>`. Surrogate characters in the range U+DC80..U+DCFF are
+   encoded to bytes 0x80..0xFF.
 
    Return a pointer to a newly allocated byte string, use :c:func:`PyMem_Free`
-   to free the memory. Return ``NULL`` on encoding error or memory allocation
-   error.
+   to free the memory.
+
+   On memory allocation failure, set *\*error_pos* to ``(size_t)-1`` and return
+   ``NULL``.
 
-   If error_pos is not ``NULL``, ``*error_pos`` is set to ``(size_t)-1`` on
-   success,  or set to the index of the invalid character on encoding error.
+   On encoding error, set *\*error_pos* to the index of the first unencodable
+   character and return ``NULL``.
 
    The :term:`filesystem encoding and error handler` are selected by
    :c:func:`PyConfig_Read`: see :c:member:`~PyConfig.filesystem_encoding` and

Doc/c-api/unicode.rst

@@ -168,8 +168,14 @@ access to internal read-only data of Unicode objects:
    The function performs no checks for any of its requirements,
    and is intended for usage in loops.
 
+   While :class:`str` objects are usually immutable in Python, this special C API allows
+   mutating a fresh :class:`str` object if the string has not been "used" yet.
+
    .. versionadded:: 3.3
 
+   .. soft-deprecated:: next
+      Use the :c:type:`PyUnicodeWriter` API instead.
+
 
 .. c:function:: Py_UCS4 PyUnicode_READ(int kind, void *data, Py_ssize_t index)
 
@@ -407,9 +413,15 @@ APIs:
    using the :c:type:`PyUnicodeWriter` API, or one of the ``PyUnicode_From*``
    functions below.
 
+   While :class:`str` objects are usually immutable in Python, this special C API
+   returns a :class:`str` object that can be mutated, except if *size* is zero, in which
+   case it returns the immutable empty string constant.
 
    .. versionadded:: 3.3
 
+   .. soft-deprecated:: next
+      Use the :c:type:`PyUnicodeWriter` API instead.
+
 
 .. c:function:: PyObject* PyUnicode_FromKindAndData(int kind, const void *buffer, \
                                                     Py_ssize_t size)
@@ -754,11 +766,16 @@ APIs:
    possible.  Returns ``-1`` and sets an exception on error, otherwise returns
    the number of copied characters.
 
-   The string must not have been “used” yet.
+   While :class:`str` objects are usually immutable in Python, this special C API allows
+   mutating a fresh :class:`str` object if the string has not been "used" yet.
+
    See :c:func:`PyUnicode_New` for details.
 
    .. versionadded:: 3.3
 
+   .. soft-deprecated:: next
+      Use the :c:type:`PyUnicodeWriter` API instead.
+
 
 .. c:function:: int PyUnicode_Resize(PyObject **unicode, Py_ssize_t length);
 
@@ -774,6 +791,14 @@ APIs:
    The function doesn't check string content, the result may not be a
    string in canonical representation.
 
+   While :class:`str` objects are usually immutable in Python, this special C API
+   can resize a :class:`str` object in-place if the string has not been "used" yet.
+   It returns a :class:`str` object which can be mutated, except if *size* is zero, in
+   which case it returns the immutable empty string constant.
+
+   .. soft-deprecated:: next
+      Use the :c:type:`PyUnicodeWriter` API instead.
+
 
 .. c:function:: Py_ssize_t PyUnicode_Fill(PyObject *unicode, Py_ssize_t start, \
                         Py_ssize_t length, Py_UCS4 fill_char)
@@ -784,14 +809,19 @@ APIs:
    Fail if *fill_char* is bigger than the string maximum character, or if the
    string has more than 1 reference.
 
-   The string must not have been “used” yet.
-   See :c:func:`PyUnicode_New` for details.
-
    Return the number of written characters, or return ``-1`` and raise an
    exception on error.
 
+   While :class:`str` objects are usually immutable in Python, this special C API allows
+   mutating a fresh :class:`str` object if the string has not been "used" yet.
+
+   See :c:func:`PyUnicode_New` for details.
+
    .. versionadded:: 3.3
 
+   .. soft-deprecated:: next
+      Use the :c:type:`PyUnicodeWriter` API instead.
+
 
 .. c:function:: int PyUnicode_WriteChar(PyObject *unicode, Py_ssize_t index, \
                                         Py_UCS4 character)
@@ -804,11 +834,16 @@ APIs:
    See :c:func:`PyUnicode_WRITE` for a version that skips these checks,
    making them your responsibility.
 
-   The string must not have been “used” yet.
+   While :class:`str` objects are usually immutable in Python, this special C API allows
+   mutating a fresh :class:`str` object if the string has not been "used" yet.
+
    See :c:func:`PyUnicode_New` for details.
 
    .. versionadded:: 3.3
 
+   .. soft-deprecated:: next
+      Use the :c:type:`PyUnicodeWriter` API instead.
+
 
 .. c:function:: Py_UCS4 PyUnicode_ReadChar(PyObject *unicode, Py_ssize_t index)
 

Doc/library/ctypes.rst

@@ -1755,8 +1755,9 @@ These prefabricated library loaders are available:
       :c:expr:`int`, which is of course not always the truth, so you have to assign
       the correct :attr:`!restype` attribute to use these functions.
 
-      Note that if the Python interpreter is statically linked, this will be
-      ``None``, as ``dlopen`` is not possible in this case.
+      .. note::
+
+         If the Python interpreter is statically linked, this may be ``None``.
 
 .. audit-event:: ctypes.dlopen name ctypes.LibraryLoader
 

Doc/using/configure.rst

@@ -1026,6 +1026,10 @@ Debug options
 
    Enable MemorySanitizer allocation error detector, ``msan`` (default is no).
 
+   MSan reports false positives for memory initialized by libraries that are
+   not built with MSan, so either build all dependencies with MSan or disable
+   the extension modules that use them in :file:`Modules/Setup.local`.
+
    .. versionadded:: 3.6
 
 .. option:: --with-undefined-behavior-sanitizer

Doc/whatsnew/3.16.rst

@@ -1224,6 +1224,13 @@ Deprecated C APIs
   :c:func:`PyModule_GetFilenameObject` instead is still recommended.
   (Contributed by Victor Stinner in :gh:`154757`.)
 
+* Soft deprecate functions modifying Unicode strings:
+  :c:func:`PyUnicode_New`, :c:func:`PyUnicode_CopyCharacters`,
+  :c:func:`PyUnicode_Fill`, :c:func:`PyUnicode_Resize`,
+  :c:func:`PyUnicode_WRITE` and :c:func:`PyUnicode_WriteChar`.
+  Use the safer :c:type:`PyUnicodeWriter` API instead.
+  (Contributed by Victor Stinner in :gh:`157710`.)
+
 .. Add C API deprecations above alphabetically, not here at the end.
 
 .. include:: ../deprecations/c-api-pending-removal-in-3.18.rst

Include/internal/pycore_fileutils.h

@@ -24,27 +24,22 @@ extern "C" {
 PyAPI_FUNC(_Py_error_handler) _Py_GetErrorHandler(const char *errors);
 
 // Export for '_testinternalcapi' shared extension
-PyAPI_FUNC(int) _Py_DecodeLocaleEx(
+PyAPI_FUNC(int) _Py_DecodeLocale(
     const char *arg,
     wchar_t **wstr,
     size_t *wlen,
-    const char **reason,
     int current_locale,
     _Py_error_handler errors);
 
 // Export for '_testinternalcapi' shared extension
-PyAPI_FUNC(int) _Py_EncodeLocaleEx(
+PyAPI_FUNC(int) _Py_EncodeLocale(
     const wchar_t *text,
     char **str,
+    size_t *output_length,
     size_t *error_pos,
-    const char **reason,
     int current_locale,
     _Py_error_handler errors);
 
-extern char* _Py_EncodeLocaleRaw(
-    const wchar_t *text,
-    size_t *error_pos);
-
 extern PyObject* _Py_device_encoding(int);
 
 #if defined(MS_WINDOWS) || defined(__APPLE__)
@@ -190,19 +185,23 @@ extern int _Py_open_osfhandle(void *handle, int flags);
      ? _PyStatus_ERR("cannot decode " NAME) \
      : _PyStatus_NO_MEMORY()
 
-extern int _Py_DecodeUTF8Ex(
+#define _Py_CODEC_MEMORY_ERROR -1
+#define _Py_CODEC_DECODE_ERROR -2
+#define _Py_CODEC_ENCODE_ERROR -2
+#define _Py_CODEC_UNSUPPORTED_ERROR_HANDLER -3
+
+extern int _Py_DecodeUTF8(
     const char *arg,
     Py_ssize_t arglen,
     wchar_t **wstr,
     size_t *wlen,
-    const char **reason,
     _Py_error_handler errors);
 
-extern int _Py_EncodeUTF8Ex(
+extern int _Py_EncodeUTF8(
     const wchar_t *text,
     char **str,
+    size_t *output_length,
     size_t *error_pos,
-    const char **reason,
     int raw_malloc,
     _Py_error_handler errors);
 

Include/internal/pycore_global_objects_fini_generated.h

@@ -54,6 +54,9 @@ struct _Py_global_strings {
         STRUCT_FOR_STR(native, "<native>")
         STRUCT_FOR_STR(str_replace_inf, "1e309")
         STRUCT_FOR_STR(type_params, ".type_params")
+        STRUCT_FOR_STR(unknown_file, "<unknown file>")
+        STRUCT_FOR_STR(unknown_function, "<unknown function>")
+        STRUCT_FOR_STR(unreadable_frame, "<unreadable frame>")
         STRUCT_FOR_STR(utf_8, "utf-8")
     } literals;