Skip to content

Commit f1f155b

Browse files
authored
Merge branch 'main' into ios-testbed-space-path-docs
2 parents c9d9977 + d8bc666 commit f1f155b

14 files changed

Lines changed: 486 additions & 445 deletions

File tree

‎Lib/test/test_capi/test_object.py‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -354,6 +354,83 @@ def test_pyobject_dump(self):
354354
output = self.pyobject_dump(NULL)
355355
self.assertRegex(output, r'<object at .* is freed>')
356356

357+
def test_object_clear_managed_dict(self):
358+
# PyObject_ClearManagedDict
359+
clear_managed_dict = _testcapi.pyobject_clear_managed_dict
360+
ManagedDictType = _testcapi.ManagedDictType
361+
362+
obj = ManagedDictType()
363+
obj.value = 'first'
364+
# asserting `__dict__` is a must to materialize the value before clean:
365+
self.assertEqual(obj.__dict__, {'value': 'first'})
366+
self.assertIs(type(obj.__dict__), dict)
367+
clear_managed_dict(obj)
368+
self.assertEqual(obj.__dict__, {})
369+
self.assertIs(type(obj.__dict__), dict)
370+
371+
# big object:
372+
obj = ManagedDictType()
373+
existing_dict = {f'attr_{x}': x for x in range(100)}
374+
obj.__dict__ = existing_dict
375+
self.assertIs(obj.__dict__, existing_dict)
376+
clear_managed_dict(obj)
377+
self.assertEqual(obj.__dict__, {})
378+
self.assertIs(type(obj.__dict__), dict)
379+
380+
# empty dict:
381+
obj = ManagedDictType()
382+
self.assertEqual(obj.__dict__, {})
383+
self.assertIs(type(obj.__dict__), dict)
384+
clear_managed_dict(obj)
385+
self.assertEqual(obj.__dict__, {})
386+
self.assertIs(type(obj.__dict__), dict)
387+
388+
# double clear:
389+
obj = ManagedDictType()
390+
obj.value = 'first'
391+
self.assertEqual(obj.__dict__, {'value': 'first'})
392+
clear_managed_dict(obj)
393+
clear_managed_dict(obj)
394+
self.assertEqual(obj.__dict__, {})
395+
self.assertIs(type(obj.__dict__), dict)
396+
397+
# unmaterialized:
398+
obj = ManagedDictType()
399+
obj.value = 'first'
400+
clear_managed_dict(obj)
401+
self.assertEqual(obj.__dict__, {})
402+
self.assertIs(type(obj.__dict__), dict)
403+
404+
# CRASHES: clear_managed_dict(NULL)
405+
406+
@threading_helper.requires_working_threading()
407+
@support.nomemtest
408+
def test_object_clear_managed_dict_ft_memory_error(self):
409+
# gh-152254: Test PyObject_ClearManagedDict() error.
410+
# The setup must contain: object owned by another thread
411+
# and the attempt to clear the managed dict with no memory left.
412+
clear_managed_dict = _testcapi.pyobject_clear_managed_dict
413+
ManagedDictType = _testcapi.ManagedDictType
414+
415+
obj = ManagedDictType()
416+
obj.value = 'first'
417+
self.assertEqual(len(obj.__dict__), 1)
418+
419+
def worker():
420+
# This needs to be in a separate thread for the crash to happen:
421+
with support.catch_unraisable_exception() as ex:
422+
# `PyObject_ClearManagedDict()` used to crash:
423+
_testcapi.call_with_nomemory(0, 1, clear_managed_dict, obj)
424+
self.assertIsNotNone(ex.unraisable)
425+
self.assertIs(ex.unraisable.exc_type, MemoryError)
426+
self.assertIn('managed dict', ex.unraisable.err_msg)
427+
self.assertEqual(obj.__dict__, {})
428+
self.assertIs(type(obj.__dict__), dict)
429+
430+
threading_helper.run_concurrently([worker])
431+
self.assertEqual(obj.__dict__, {})
432+
self.assertIs(type(obj.__dict__), dict)
433+
357434

358435
class RefTracerTest(unittest.TestCase):
359436
@support.requires_resource('cpu')

‎Lib/test/test_external_inspection.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4687,6 +4687,28 @@ def test_dead_process_raises_not_degrades(self):
46874687
else:
46884688
self.fail("ProcessLookupError never raised for dead process")
46894689

4690+
def test_dead_process_before_first_async_read(self):
4691+
script_body = """\
4692+
import asyncio
4693+
sock.sendall(b"ready")
4694+
sock.recv(16)
4695+
"""
4696+
with self._target_process(script_body) as (p, client_socket, _):
4697+
_wait_for_signal(client_socket, b"ready")
4698+
unwinders = [
4699+
RemoteUnwinder(p.pid, debug=debug)
4700+
for debug in (False, True)
4701+
]
4702+
4703+
p.kill()
4704+
p.wait()
4705+
4706+
for debug, unwinder in zip((False, True), unwinders):
4707+
for method in ("get_async_stack_trace", "get_all_awaited_by"):
4708+
with self.subTest(debug=debug, method=method):
4709+
with self.assertRaises(ProcessLookupError):
4710+
getattr(unwinder, method)()
4711+
46904712

46914713
@requires_remote_subprocess_debugging()
46924714
class TestFrameChainLimits(RemoteInspectionTestBase):

‎Lib/test/test_time.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -562,8 +562,8 @@ def test_tzset(self):
562562
# on some operating systems (e.g. FreeBSD), which is wrong. See for
563563
# example this bug:
564564
# http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=93810
565-
self.assertIn(time.tzname[0], ('AEST' 'EST'), time.tzname[0])
566-
self.assertTrue(time.tzname[1] in ('AEDT', 'EDT'), str(time.tzname[1]))
565+
self.assertIn(time.tzname[0], ('AEST', 'EST'))
566+
self.assertIn(time.tzname[1], ('AEDT', 'EDT'))
567567
self.assertEqual(len(time.tzname), 2)
568568
self.assertEqual(time.daylight, 1)
569569
self.assertEqual(time.timezone, -36000)

‎Lib/test/test_venv.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,28 @@ def test_prefixes(self):
302302
self.assertEqual(pathlib.Path(out.strip().decode()),
303303
pathlib.Path(expected), prefix)
304304

305+
# gh-157917: the "\\?\" extended-length prefix must not leak into the prefix
306+
@requireVenvCreate
307+
@unittest.skipUnless(os.name == 'nt', 'only relevant on Windows')
308+
def test_prefixes_extended_length_path(self):
309+
"""
310+
Test that invoking a venv through a "\\?\" extended-length path does
311+
not leave that prefix behind in the prefixes. Windows does not
312+
resolve relative components (such as ".." or "/") in such paths, so
313+
they break tools that join them onto sysconfig-derived paths.
314+
"""
315+
rmtree(self.env_dir)
316+
self.run_with_capture(venv.create, self.env_dir)
317+
envpy = os.path.join('\\\\?\\' + self.env_dir, self.bindir, self.exe)
318+
for prefix in ('prefix', 'exec_prefix'):
319+
with self.subTest(prefix):
320+
out, err = check_output(
321+
[envpy, '-c', 'import sys; print(sys.%s)' % prefix],
322+
encoding='utf-8')
323+
self.assertFalse(out.strip().startswith('\\\\?\\'), out)
324+
self.assertEqual(pathlib.Path(out.strip()),
325+
pathlib.Path(self.env_dir), prefix)
326+
305327
@requireVenvCreate
306328
def test_sysconfig(self):
307329
"""
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fix a crash during the clean up of managed dict objects under a memory
2+
pressure in :c:func:`PyObject_ClearManagedDict`.
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
Fix :attr:`sys.prefix`, :attr:`sys.exec_prefix` and :attr:`sys.executable`
2+
retaining the ``\\?\`` extended-length path prefix in a virtual environment
3+
when the interpreter is invoked through such a path on Windows. The venv
4+
launcher reports its own path through ``GetModuleFileNameW``, which preserves
5+
the prefix, and that value was used verbatim. Windows does not resolve
6+
relative path components in extended-length paths, so the leaked prefix made
7+
tools that join ``..`` components onto :mod:`sysconfig` paths fail with
8+
``OSError: [Errno 22]``. The prefix is now stripped again, except for paths
9+
that are still too long to be used without it.

‎Modules/_remote_debugging/_remote_debugging.h‎

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@ extern "C" {
5656
# if TARGET_OS_OSX
5757
# include <libproc.h>
5858
# include <sys/types.h>
59-
# define MAX_NATIVE_THREADS 4096
6059
# endif
6160
#endif
6261

@@ -104,18 +103,7 @@ typedef struct {
104103

105104
#ifdef MS_WINDOWS
106105
#define STATUS_SUCCESS ((NTSTATUS)0x00000000L)
107-
#define STATUS_INFO_LENGTH_MISMATCH ((NTSTATUS)0xC0000004L)
108106
#define STATUS_NO_MORE_ENTRIES ((NTSTATUS)0x8000001AL)
109-
typedef enum _WIN32_THREADSTATE {
110-
WIN32_THREADSTATE_INITIALIZED = 0,
111-
WIN32_THREADSTATE_READY = 1,
112-
WIN32_THREADSTATE_RUNNING = 2,
113-
WIN32_THREADSTATE_STANDBY = 3,
114-
WIN32_THREADSTATE_TERMINATED = 4,
115-
WIN32_THREADSTATE_WAITING = 5,
116-
WIN32_THREADSTATE_TRANSITION = 6,
117-
WIN32_THREADSTATE_UNKNOWN = 7
118-
} WIN32_THREADSTATE;
119107
#endif
120108

121109
/* ============================================================================
@@ -387,14 +375,6 @@ typedef struct {
387375
#ifdef Py_GIL_DISABLED
388376
uint32_t tlbc_generation;
389377
_Py_hashtable_t *tlbc_cache;
390-
#endif
391-
#ifdef __APPLE__
392-
uint64_t thread_id_offset;
393-
int thread_id_offset_initialized;
394-
#endif
395-
#ifdef MS_WINDOWS
396-
PVOID win_process_buffer;
397-
ULONG win_process_buffer_size;
398378
#endif
399379
// Thread stopping state (only on platforms that support it)
400380
#ifdef Py_REMOTE_DEBUG_SUPPORTS_BLOCKING
@@ -681,8 +661,6 @@ extern int find_running_frame(
681661
uintptr_t *frame
682662
);
683663

684-
extern int get_thread_status(RemoteUnwinderObject *unwinder, uint64_t tid, uint64_t pthread_id);
685-
686664
extern PyObject* unwind_stack_for_thread(
687665
RemoteUnwinderObject *unwinder,
688666
uintptr_t *current_tstate,

‎Modules/_remote_debugging/asyncio.c‎

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ _Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle)
2222
address = search_windows_map_for_section(handle, "AsyncioD", L"_asyncio",
2323
NULL);
2424
if (address == 0) {
25-
if (!_Py_RemoteDebug_HasPermissionError()) {
25+
if (!_Py_RemoteDebug_IsFatalReadError()) {
2626
PyObject *exc = PyErr_GetRaisedException();
2727
PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process.");
2828
_PyErr_ChainExceptions1(exc);
@@ -33,23 +33,16 @@ _Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle)
3333
address = search_linux_map_for_section(handle, "AsyncioDebug", "python",
3434
NULL);
3535
if (address == 0) {
36-
if (!_Py_RemoteDebug_HasPermissionError()) {
36+
if (!_Py_RemoteDebug_IsFatalReadError()) {
3737
PyObject *exc = PyErr_GetRaisedException();
3838
PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process.");
3939
_PyErr_ChainExceptions1(exc);
4040
}
4141
}
4242
#elif defined(__APPLE__) && TARGET_OS_OSX
43-
// On macOS, try libpython first, then fall back to python
44-
address = search_map_for_section(handle, "AsyncioDebug", "libpython",
45-
NULL);
46-
if (address == 0 && !_Py_RemoteDebug_HasPermissionError()) {
47-
PyErr_Clear();
48-
address = search_map_for_section(handle, "AsyncioDebug", "python",
49-
NULL);
50-
}
43+
address = search_loaded_images_for_section(handle, "AsyncioDebug", NULL);
5144
if (address == 0) {
52-
if (!_Py_RemoteDebug_HasPermissionError()) {
45+
if (!_Py_RemoteDebug_IsFatalReadError()) {
5346
PyObject *exc = PyErr_GetRaisedException();
5447
PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process.");
5548
_PyErr_ChainExceptions1(exc);
@@ -99,12 +92,22 @@ ensure_async_debug_offsets(RemoteUnwinderObject *unwinder)
9992
return -1;
10093
}
10194
if (result < 0) {
102-
if (!_Py_RemoteDebug_HasPermissionError()) {
103-
PyErr_Clear();
104-
PyErr_SetString(PyExc_RuntimeError, "AsyncioDebug section not available");
105-
set_exception_cause(unwinder, PyExc_RuntimeError,
106-
"AsyncioDebug section unavailable - asyncio module may not be loaded in target process");
95+
if (_Py_RemoteDebug_IsFatalReadError()) {
96+
return -1;
10797
}
98+
PyErr_Clear();
99+
/* Image discovery can fail without reporting that the target exited.
100+
Read a known runtime address before reporting unavailable asyncio. */
101+
char runtime_byte;
102+
if (_Py_RemoteDebug_ReadRemoteMemory(&unwinder->handle,
103+
unwinder->runtime_start_address, sizeof(runtime_byte),
104+
&runtime_byte) < 0)
105+
{
106+
return -1;
107+
}
108+
PyErr_SetString(PyExc_RuntimeError, "AsyncioDebug section not available");
109+
set_exception_cause(unwinder, PyExc_RuntimeError,
110+
"AsyncioDebug section unavailable - asyncio module may not be loaded in target process");
108111
return -1;
109112
}
110113

‎Modules/_remote_debugging/module.c‎

Lines changed: 2 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -384,12 +384,6 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self,
384384
return -1;
385385
}
386386

387-
self->runtime_start_address = _Py_RemoteDebug_GetPyRuntimeAddress(&self->handle);
388-
if (self->runtime_start_address == 0) {
389-
set_exception_cause(self, PyExc_RuntimeError, "Failed to get Python runtime address");
390-
return -1;
391-
}
392-
393387
if (_Py_RemoteDebug_ReadDebugOffsets(&self->handle,
394388
&self->runtime_start_address,
395389
&self->debug_offsets) < 0)
@@ -404,20 +398,8 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self,
404398
return -1;
405399
}
406400

407-
// Try to read async debug offsets, but don't fail if they're not available
408-
self->async_debug_offsets_available = 1;
409-
int async_debug_result = read_async_debug(self);
410-
if (async_debug_result == PY_REMOTE_DEBUG_INVALID_ASYNC_DEBUG_OFFSETS) {
411-
return -1;
412-
}
413-
if (async_debug_result < 0) {
414-
if (_Py_RemoteDebug_HasPermissionError()) {
415-
return -1;
416-
}
417-
PyErr_Clear();
418-
memset(&self->async_debug_offsets, 0, sizeof(self->async_debug_offsets));
419-
self->async_debug_offsets_available = 0;
420-
}
401+
self->async_debug_offsets_available = 0;
402+
memset(&self->async_debug_offsets, 0, sizeof(self->async_debug_offsets));
421403

422404
if (populate_initial_state_data(all_threads, self, self->runtime_start_address,
423405
&self->interpreter_addr ,&self->tstate_addr) < 0)
@@ -456,15 +438,6 @@ _remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self,
456438
}
457439
#endif
458440

459-
#if defined(__APPLE__)
460-
self->thread_id_offset = 0;
461-
self->thread_id_offset_initialized = 0;
462-
#endif
463-
464-
#ifdef MS_WINDOWS
465-
self->win_process_buffer = NULL;
466-
self->win_process_buffer_size = 0;
467-
#endif
468441
#ifdef __linux__
469442
self->thread_tids = NULL;
470443
self->thread_tids_capacity = 0;
@@ -1347,11 +1320,6 @@ RemoteUnwinder_dealloc(PyObject *op)
13471320
if (self->code_object_cache) {
13481321
_Py_hashtable_destroy(self->code_object_cache);
13491322
}
1350-
#ifdef MS_WINDOWS
1351-
if (self->win_process_buffer != NULL) {
1352-
PyMem_Free(self->win_process_buffer);
1353-
}
1354-
#endif
13551323

13561324
#ifdef Py_GIL_DISABLED
13571325
if (self->tlbc_cache) {
@@ -1406,11 +1374,6 @@ init_runtime_offsets(RuntimeOffsets *offsets, pid_t pid, int debug)
14061374
set_exception_cause(offsets, PyExc_RuntimeError, "Failed to initialize process handle");
14071375
return -1;
14081376
}
1409-
offsets->runtime_start_address = _Py_RemoteDebug_GetPyRuntimeAddress(&offsets->handle);
1410-
if (offsets->runtime_start_address == 0) {
1411-
set_exception_cause(offsets, PyExc_RuntimeError, "Failed to get Python runtime address");
1412-
goto error;
1413-
}
14141377
if (_Py_RemoteDebug_ReadDebugOffsets(&offsets->handle,
14151378
&offsets->runtime_start_address,
14161379
&offsets->debug_offsets) < 0)

0 commit comments

Comments
 (0)