Merge remote-tracking branch 'upstream/main' into tachyon-land-157871

Author: pablogsal

Doc/c-api/unicode.rst

@@ -168,8 +168,14 @@ access to internal read-only data of Unicode objects:
    The function performs no checks for any of its requirements,
    and is intended for usage in loops.
 
+   While :class:`str` objects are usually immutable in Python, this special C API allows
+   mutating a fresh :class:`str` object if the string has not been "used" yet.
+
    .. versionadded:: 3.3
 
+   .. soft-deprecated:: next
+      Use the :c:type:`PyUnicodeWriter` API instead.
+
 
 .. c:function:: Py_UCS4 PyUnicode_READ(int kind, void *data, Py_ssize_t index)
 
@@ -407,9 +413,15 @@ APIs:
    using the :c:type:`PyUnicodeWriter` API, or one of the ``PyUnicode_From*``
    functions below.
 
+   While :class:`str` objects are usually immutable in Python, this special C API
+   returns a :class:`str` object that can be mutated, except if *size* is zero, in which
+   case it returns the immutable empty string constant.
 
    .. versionadded:: 3.3
 
+   .. soft-deprecated:: next
+      Use the :c:type:`PyUnicodeWriter` API instead.
+
 
 .. c:function:: PyObject* PyUnicode_FromKindAndData(int kind, const void *buffer, \
                                                     Py_ssize_t size)
@@ -754,11 +766,16 @@ APIs:
    possible.  Returns ``-1`` and sets an exception on error, otherwise returns
    the number of copied characters.
 
-   The string must not have been “used” yet.
+   While :class:`str` objects are usually immutable in Python, this special C API allows
+   mutating a fresh :class:`str` object if the string has not been "used" yet.
+
    See :c:func:`PyUnicode_New` for details.
 
    .. versionadded:: 3.3
 
+   .. soft-deprecated:: next
+      Use the :c:type:`PyUnicodeWriter` API instead.
+
 
 .. c:function:: int PyUnicode_Resize(PyObject **unicode, Py_ssize_t length);
 
@@ -774,6 +791,14 @@ APIs:
    The function doesn't check string content, the result may not be a
    string in canonical representation.
 
+   While :class:`str` objects are usually immutable in Python, this special C API
+   can resize a :class:`str` object in-place if the string has not been "used" yet.
+   It returns a :class:`str` object which can be mutated, except if *size* is zero, in
+   which case it returns the immutable empty string constant.
+
+   .. soft-deprecated:: next
+      Use the :c:type:`PyUnicodeWriter` API instead.
+
 
 .. c:function:: Py_ssize_t PyUnicode_Fill(PyObject *unicode, Py_ssize_t start, \
                         Py_ssize_t length, Py_UCS4 fill_char)
@@ -784,14 +809,19 @@ APIs:
    Fail if *fill_char* is bigger than the string maximum character, or if the
    string has more than 1 reference.
 
-   The string must not have been “used” yet.
-   See :c:func:`PyUnicode_New` for details.
-
    Return the number of written characters, or return ``-1`` and raise an
    exception on error.
 
+   While :class:`str` objects are usually immutable in Python, this special C API allows
+   mutating a fresh :class:`str` object if the string has not been "used" yet.
+
+   See :c:func:`PyUnicode_New` for details.
+
    .. versionadded:: 3.3
 
+   .. soft-deprecated:: next
+      Use the :c:type:`PyUnicodeWriter` API instead.
+
 
 .. c:function:: int PyUnicode_WriteChar(PyObject *unicode, Py_ssize_t index, \
                                         Py_UCS4 character)
@@ -804,11 +834,16 @@ APIs:
    See :c:func:`PyUnicode_WRITE` for a version that skips these checks,
    making them your responsibility.
 
-   The string must not have been “used” yet.
+   While :class:`str` objects are usually immutable in Python, this special C API allows
+   mutating a fresh :class:`str` object if the string has not been "used" yet.
+
    See :c:func:`PyUnicode_New` for details.
 
    .. versionadded:: 3.3
 
+   .. soft-deprecated:: next
+      Use the :c:type:`PyUnicodeWriter` API instead.
+
 
 .. c:function:: Py_UCS4 PyUnicode_ReadChar(PyObject *unicode, Py_ssize_t index)
 

Doc/whatsnew/3.16.rst

@@ -1224,6 +1224,13 @@ Deprecated C APIs
   :c:func:`PyModule_GetFilenameObject` instead is still recommended.
   (Contributed by Victor Stinner in :gh:`154757`.)
 
+* Soft deprecate functions modifying Unicode strings:
+  :c:func:`PyUnicode_New`, :c:func:`PyUnicode_CopyCharacters`,
+  :c:func:`PyUnicode_Fill`, :c:func:`PyUnicode_Resize`,
+  :c:func:`PyUnicode_WRITE` and :c:func:`PyUnicode_WriteChar`.
+  Use the safer :c:type:`PyUnicodeWriter` API instead.
+  (Contributed by Victor Stinner in :gh:`157710`.)
+
 .. Add C API deprecations above alphabetically, not here at the end.
 
 .. include:: ../deprecations/c-api-pending-removal-in-3.18.rst

Include/pymacro.h

@@ -199,19 +199,23 @@
         } while(0)
 #endif
 
-/* Get the number of elements in a visible array
-
-   This does not work on pointers, or arrays declared as [], or function
-   parameters. With correct compiler support, such usage will cause a build
-   error (see Py_BUILD_ASSERT_EXPR).
-
-   Written by Rusty Russell, public domain, http://ccodearchive.net/
-
-   Requires at GCC 3.1+ */
-#if (defined(__GNUC__) && !defined(__STRICT_ANSI__) && \
-    (((__GNUC__ == 3) && (__GNUC_MINOR__ >= 1)) || (__GNUC__ >= 4)))
-/* Two gcc extensions.
-   &a[0] degrades to a pointer: a different type from an array */
+// Get the number of elements in a visible array.
+//
+// This does not work on pointers, or arrays declared as [], or function
+// parameters. With correct compiler support, such usage will cause a build
+// error (see Py_BUILD_ASSERT_EXPR).
+//
+// Written by Rusty Russell, public domain, http://ccodearchive.net/
+//
+// Require GCC 4 (it works on GCC 3.1).
+//
+// Two GCC extensions: &a[0] degrades to a pointer, a different type from an
+// array.
+//
+// gh-158810: Do not use __builtin_types_compatible_p() in strict C ANSI mode
+// and on C++.
+#if (defined(__GNUC__) && __GNUC__ >= 4 \
+        && !defined(__STRICT_ANSI__) && !defined(__cplusplus))
 #define Py_ARRAY_LENGTH(array) \
     (sizeof(array) / sizeof((array)[0]) \
      + Py_BUILD_ASSERT_EXPR(!__builtin_types_compatible_p(typeof(array), \

Lib/profiling/sampling/_sync_coordinator.py

@@ -168,6 +168,11 @@ def _execute_script(script_path: str, script_args: List[str], cwd: str) -> None:
     if not os.path.isfile(script_path):
         raise TargetError(f"Script not found: {script_path}")
 
+    script_dir = os.path.dirname(os.path.realpath(script_path))
+    if script_dir in sys.path:
+        sys.path.remove(script_dir)
+    sys.path.insert(0, script_dir)
+
     # Replace sys.argv to match original script call
     sys.argv = [script_path] + script_args
 

Lib/test/test_capi/test_unicode.py

@@ -25,6 +25,7 @@
 # Maximum invalid character which fits into 32-bit Py_UCS4
 MAX_INVALID_CHAR = 0xFFFF_FFFF
 NULL = None
+USED_STR_ERROR = 'Cannot modify a string currently used'
 
 class Str(str):
     pass
@@ -76,9 +77,27 @@ def test_checkexact(self):
         # Test PyUnicode_CheckExact()
         self._test_check(_testlimitedcapi.unicode_checkexact, exact=True)
 
+    def assert_is_mutable(self, result, refcnt):
+        # Check that result is a "mutable" Unicode string
+        self.assertEqual(refcnt, 1)
+        self.assertFalse(sys._is_immortal(result))
+
+    def assert_is_empty_singleton(self, result):
+        # Check that result is the empty string singleton
+        self.assertEqual(result, '')
+        self.assertTrue(sys._is_immortal(result))
+
     def test_new(self):
         """Test PyUnicode_New()"""
-        new = _testcapi.unicode_new
+        _unicode_new = _testcapi.unicode_new
+
+        def new(size, maxchar):
+            result = _unicode_new(size, maxchar)
+            if size != 0:
+                self.assert_is_mutable(result, sys.getrefcount(result))
+            else:
+                self.assert_is_empty_singleton(result)
+            return result
 
         for maxchar in 0, 0x61, 0xa1, 0x4f60, 0x1f600, 0x10ffff:
             self.assertEqual(new(0, maxchar), '')
@@ -123,6 +142,10 @@ def test_fill(self):
                         self.assertEqual(fill(to, start, length, fill_char),
                                         (expected, filled))
 
+        # A string with 2 references cannot be modified
+        with self.assertRaisesRegex(SystemError, USED_STR_ERROR):
+            fill('abc', 0, 3, ord('x'), incref=True)
+
         s = strings[0]
         self.assertRaises(IndexError, fill, s, -1, 0, 0x78)
         self.assertRaises(IndexError, fill, s, PY_SSIZE_T_MIN, 0, 0x78)
@@ -162,7 +185,12 @@ def _test_writechar(self, writechar, *, check):
 
     def test_writechar(self):
         """Test PyUnicode_WriteChar()"""
-        self._test_writechar(_testlimitedcapi.unicode_writechar, check=True)
+        writechar = _testlimitedcapi.unicode_writechar
+        self._test_writechar(writechar, check=True)
+
+        # A string with 2 references cannot be modified
+        with self.assertRaisesRegex(SystemError, USED_STR_ERROR):
+            writechar('abc', 1, ord('x'), incref=True)
 
     def test_write_macro(self):
         """Test PyUnicode_WRITE()"""
@@ -187,18 +215,16 @@ def resize(s, length, new=True, compute_hash=False):
                 self.assertFalse(is_new_obj)
             elif length == 0:
                 # Get the empty Unicode string
-                self.assertEqual(result, '')
-                self.assertTrue(sys._is_immortal(result))
+                self.assert_is_empty_singleton(result)
                 self.assertTrue(is_new_obj)
             elif (not new) or compute_hash:
                 # Get a fresh copy
-                self.assertEqual(refcnt, 1)
+                self.assert_is_mutable(result, refcnt)
                 self.assertTrue(is_new_obj)
-                self.assertFalse(sys._is_immortal(result))
             else:
                 # In-size replace can return the same address, or not.
                 # So 'is_new_obj' cannot be tested.
-                self.assertFalse(sys._is_immortal(result))
+                self.assert_is_mutable(result, refcnt)
 
             return result
 
@@ -1793,6 +1819,11 @@ def test_copycharacters(self):
         self.assertRaises(SystemError, unicode_copycharacters, s, 0, s, 0, PY_SSIZE_T_MIN)
         self.assertRaises(SystemError, unicode_copycharacters, s, 0, b'', 0, 0)
         self.assertRaises(SystemError, unicode_copycharacters, s, 0, [], 0, 0)
+
+        # A string with 2 references cannot be modified
+        with self.assertRaisesRegex(SystemError, USED_STR_ERROR):
+            unicode_copycharacters('abc', 0, 'abc', 0, 1, incref=True)
+
         # CRASHES unicode_copycharacters(s, 0, NULL, 0, 0)
         # TODO: Test PyUnicode_CopyCharacters() with non-unicode and
         # non-modifiable unicode as "to".

Lib/test/test_cext/extension.c

@@ -83,6 +83,7 @@ test_macros(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args))
 {
     PyObject *obj, *dict;
     PyObject *slots[1];
+    int small_array[] = {2, 5, 7};
 
     // test Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR()
     Py_BUILD_ASSERT(sizeof(int) == sizeof(unsigned int));
@@ -134,6 +135,9 @@ test_macros(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args))
     Py_END_CRITICAL_SECTION();
     Py_DECREF(dict);
 
+    // Test Py_ARRAY_LENGTH()
+    assert(Py_ARRAY_LENGTH(small_array) == 3);
+
     Py_RETURN_NONE;
 }
 

Lib/test/test_dict.py

@@ -116,6 +116,26 @@ def test_items(self):
         self.assertRaises(TypeError, d.items, None)
         self.assertEqual(repr(dict(a=1).items()), "dict_items([('a', 1)])")
 
+    @support.cpython_only
+    def test_item_iterator_oom(self):
+        import_helper.import_module('_testcapi')
+        from test.support.script_helper import assert_python_ok
+        code = """if 1:
+            import _testcapi
+            items = {1: 2, 3: 4}.items()
+            ballast = [(i, i) for i in range(3000)]
+            held = []
+            for start in range(1, 5):
+                _testcapi.set_nomemory(start)
+                try:
+                    held.append(iter(items))
+                except MemoryError:
+                    pass
+                finally:
+                    _testcapi.remove_mem_hooks()
+            """
+        assert_python_ok('-c', code)
+
     def test_views_mapping(self):
         mappingproxy = type(type.__dict__)
         class Dict(dict):

Lib/test/test_fstring.py

@@ -645,6 +645,20 @@ def test_unclosed_multiline_replacement_field(self):
                             f"{prefix[-1]}-string: expecting '}}' to close '{{' "
                             f"on line {lineno}")
 
+    def test_unclosed_replacement_field_quote_line(self):
+        for prefix in ('f', 't', 'rf', 'rt'):
+            for quote in ('"', "'"):
+                triple = quote * 3
+                for suffix in ('', '\nx'):
+                    source = prefix + triple + '{1' + triple + suffix
+                    with self.subTest(source=source):
+                        with self.assertRaises(SyntaxError) as cm:
+                            compile(source, '<test>', 'exec')
+                        self.assertEqual(
+                            cm.exception.msg,
+                            f"{prefix[-1]}-string: expecting '}}'")
+                        self.assertEqual(cm.exception.lineno, 1)
+
     @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
     def test_mismatched_parens(self):
         self.assertAllRaise(SyntaxError, r"closing parenthesis '\}' "

Lib/test/test_profiling/test_sampling_profiler/test_blocking.py

@@ -5,6 +5,7 @@
 import subprocess
 import sys
 import textwrap
+import time
 import unittest
 from unittest import mock
 
@@ -213,3 +214,59 @@ def test_run_blocking_exits_after_target_process_exits(self):
                 replay.returncode, 0,
                 f"stdout:\n{replay.stdout}\nstderr:\n{replay.stderr}",
             )
+
+
+@requires_remote_subprocess_debugging()
+@unittest.skipUnless(sys.platform == "win32", "Windows only")
+class TestBlockingModeSuspension(unittest.TestCase):
+    def test_all_threads_stop_before_pause_returns(self):
+        import mmap
+
+        tag = f"cpython_blocking_{os.getpid()}_{id(self)}"
+        script = textwrap.dedent(f'''
+            import mmap
+            import struct
+            import threading
+
+            memory = mmap.mmap(-1, 16, tagname={tag!r})
+
+            def worker(offset):
+                counter = 0
+                while True:
+                    counter += 1
+                    struct.pack_into("q", memory, offset, counter)
+
+            for offset in (0, 8):
+                threading.Thread(target=worker, args=(offset,), daemon=True).start()
+            _test_sock.sendall(b"working")
+            _test_sock.recv(1)
+        ''')
+        with mmap.mmap(-1, 16, tagname=tag) as memory:
+            with test_subprocess(script, wait_for_working=True) as subproc:
+                unwinder = _remote_debugging.RemoteUnwinder(
+                    subproc.process.pid, all_threads=True)
+                deadline = time.monotonic() + SHORT_TIMEOUT
+                while not all(memory[offset:offset + 8] != bytes(8)
+                              for offset in (0, 8)):
+                    self.assertLess(time.monotonic(), deadline,
+                                    "Worker threads did not start")
+                    time.sleep(0.001)
+                for _ in range(100):
+                    self.assertTrue(unwinder.pause_threads())
+                    try:
+                        before = memory[:]
+                        self.assertFalse(unwinder.pause_threads())
+                        unwinder.get_stack_trace()
+                        time.sleep(0.001)
+                        self.assertEqual(memory[:], before,
+                                         "Target memory changed while paused")
+                    finally:
+                        unwinder.resume_threads()
+                    self.assertFalse(unwinder.resume_threads())
+                before = memory[:]
+                deadline = time.monotonic() + SHORT_TIMEOUT
+                while any(memory[offset:offset + 8] == before[offset:offset + 8]
+                          for offset in (0, 8)):
+                    self.assertLess(time.monotonic(), deadline,
+                                    "Worker threads did not resume")
+                    time.sleep(0.001)