From 2e1fb0e5af37b839c4b02d6fd1bf686655a7b823 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Fri, 2 Oct 2026 20:52:25 +0100 Subject: [PATCH 1/4] Run the test suite with MSan in CI --- .github/workflows/build.yml | 3 +++ .github/workflows/reusable-san.yml | 19 ++++++++++++++++++- Lib/test/_test_multiprocessing.py | 2 ++ Lib/test/test_asyncio/test_tools.py | 6 ++++-- Lib/test/test_cext/__init__.py | 1 + Modules/_testinternalcapi.c | 6 +++--- Modules/posixmodule.c | 3 +++ Modules/socketmodule.c | 6 ++++++ 8 files changed, 40 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9ab0473f4b9c598..00d64b3df7e9654 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -547,6 +547,9 @@ jobs: - check-name: Undefined behavior sanitizer: UBSan free-threading: false + - check-name: Memory + sanitizer: MSan + free-threading: false uses: ./.github/workflows/reusable-san.yml with: sanitizer: ${{ matrix.sanitizer }} diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index ad3232743874d6b..91a26ebf22e463f 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -60,7 +60,7 @@ jobs: || '' }} - name: UBSan option setup - if: inputs.sanitizer != 'TSan' + if: inputs.sanitizer == 'UBSan' run: >- echo "UBSAN_OPTIONS=${SAN_LOG_OPTION} @@ -69,6 +69,21 @@ jobs: >> "$GITHUB_ENV" env: SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log + - name: MSan option setup + if: inputs.sanitizer == 'MSan' + run: | + echo "MSAN_OPTIONS=${SAN_LOG_OPTION} allocator_may_return_null=1" >> "$GITHUB_ENV" + # MSan reports false positives for memory initialized by libraries + # that are not built with MSan, so disable modules that use them. + # _remote_debugging is disabled because it links to libzstd directly. + { + echo '*disabled*' + echo '_bz2 _ctypes _curses _curses_panel _dbm _decimal _gdbm _hashlib' + echo '_lzma _remote_debugging _sqlite3 _ssl _tkinter _uuid _zstd' + echo 'readline zlib' + } > Modules/Setup.local + env: + SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log - name: Add ccache to PATH run: | echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" @@ -101,6 +116,8 @@ jobs: ${{ inputs.sanitizer == 'TSan' && '--with-thread-sanitizer' + || inputs.sanitizer == 'MSan' + && '--with-memory-sanitizer' || '--with-undefined-behavior-sanitizer --with-strict-overflow' }} --with-pydebug diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 46ed8843fcd0519..7292128fb78870a 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -3159,6 +3159,7 @@ def test_imap_and_imap_unordered_with_buffersize_type_validation( with self.assertRaisesRegex(expected_exception, expected_regex): method(str, range(4), buffersize=buffersize) + @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes') @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @support.subTests('method_name', ("imap", "imap_unordered")) def test_imap_and_imap_unordered_when_buffer_is_full(self, method_name): @@ -3194,6 +3195,7 @@ def produce_args(): p.terminate() p.join() + @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes') @warnings_helper.ignore_fork_in_thread_deprecation_warnings() @support.subTests('method_name', ("imap", "imap_unordered")) def test_imap_and_imap_unordered_with_buffersize_when_buffer_is_full( diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py index df934164eb9fd60..cd8e1e84fdcac27 100644 --- a/Lib/test/test_asyncio/test_tools.py +++ b/Lib/test/test_asyncio/test_tools.py @@ -1,8 +1,10 @@ import unittest -from asyncio import tools +from test.support import import_helper + +_remote_debugging = import_helper.import_module('_remote_debugging') -import _remote_debugging +from asyncio import tools def LocationInfo(lineno, end_lineno=None, col_offset=None, end_col_offset=None): diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py index c4fd2a1e044d892..9bd602ca2e4af46 100644 --- a/Lib/test/test_cext/__init__.py +++ b/Lib/test/test_cext/__init__.py @@ -192,6 +192,7 @@ def test_build(self): self.check_build('_test_cppext_internal') +@support.requires_venv_with_pip() def setUpModule(): global VENV_CONTEXT, PYTHON_EXE VENV_CONTEXT = support.setup_venv_with_pip_setuptools('env') diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index a2ce266eb35a655..45a3f0d6a155b83 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -469,7 +469,7 @@ next_frame_pointer_is_valid(uintptr_t *frame_pointer, uintptr_t *next_fp, #endif } -static PyObject * +static PyObject * _Py_NO_SANITIZE_MEMORY manual_unwind_from_fp(uintptr_t *frame_pointer) { uintptr_t stack_min = 0; @@ -2083,8 +2083,8 @@ check_pyobject_forbidden_bytes_is_freed(PyObject *self, static PyObject * check_pyobject_freed_is_freed(PyObject *self, PyObject *Py_UNUSED(args)) { - /* ASan or TSan would report an use-after-free error */ -#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER) + /* ASan, MSan or TSan would report an error. */ +#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER) || defined(_Py_MEMORY_SANITIZER) Py_RETURN_NONE; #else PyObject *op = PyObject_CallNoArgs((PyObject *)&PyBaseObject_Type); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 0a451b8a833e67e..e0ac41138b1f605 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -10138,6 +10138,9 @@ os_getlogin_impl(PyObject *module) errno = old_errno; } else { +#ifdef _Py_MEMORY_SANITIZER + __msan_unpoison(name, sizeof(name)); +#endif result = PyUnicode_DecodeFSDefault(name); } #else diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 61505c2603f22c2..bde7728634d7b8c 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -763,6 +763,9 @@ set_error(void) static PyObject * decode_error_message(const char *str) { +#ifdef _Py_MEMORY_SANITIZER + __msan_unpoison_string(str); +#endif return PyUnicode_DecodeLocale(str, "surrogateescape"); } #endif @@ -6522,6 +6525,9 @@ _socket_getservbyport_impl(PyObject *module, int port, const char *proto) PyErr_SetString(PyExc_OSError, "port/proto not found"); return NULL; } +#ifdef _Py_MEMORY_SANITIZER + __msan_unpoison_string(sp->s_name); +#endif return PyUnicode_FromString(sp->s_name); } From c01ed7365161d22c71a84fd4592156f722072b48 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Fri, 2 Oct 2026 21:35:35 +0100 Subject: [PATCH 2/4] Additional fixes --- .github/workflows/reusable-san.yml | 9 +++++---- Lib/test/test_asyncio/test_tools.py | 6 ++---- Modules/_remote_debugging/binary_io_reader.c | 5 +++++ Modules/_remote_debugging/binary_io_writer.c | 6 ++++++ Python/instrumentation.c | 1 + 5 files changed, 19 insertions(+), 8 deletions(-) diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index 91a26ebf22e463f..8d19d3a5ba92115 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -75,12 +75,11 @@ jobs: echo "MSAN_OPTIONS=${SAN_LOG_OPTION} allocator_may_return_null=1" >> "$GITHUB_ENV" # MSan reports false positives for memory initialized by libraries # that are not built with MSan, so disable modules that use them. - # _remote_debugging is disabled because it links to libzstd directly. + # _remote_debugging links to libzstd directly, but we unpoision the memory. { echo '*disabled*' echo '_bz2 _ctypes _curses _curses_panel _dbm _decimal _gdbm _hashlib' - echo '_lzma _remote_debugging _sqlite3 _ssl _tkinter _uuid _zstd' - echo 'readline zlib' + echo '_lzma _sqlite3 _ssl _tkinter _uuid _zstd readline zlib' } > Modules/Setup.local env: SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log @@ -108,6 +107,8 @@ jobs: # gh-157958: -O2 instead of the pydebug default -Og to avoid a clang 21 # compile-time blowup on some interpreter files. # (https://github.com/llvm/llvm-project/issues/179695) + # MSan uses --with-assertions instead of --with-pydebug because its + # hooks on the Python memory allocators hide uninitialized reads. - name: Configure CPython run: >- ./configure @@ -120,7 +121,7 @@ jobs: && '--with-memory-sanitizer' || '--with-undefined-behavior-sanitizer --with-strict-overflow' }} - --with-pydebug + ${{ inputs.sanitizer == 'MSan' && '--with-assertions' || '--with-pydebug' }} ${{ inputs.sanitizer == 'TSan' && '--with-openssl="$OPENSSL_DIR" --with-openssl-rpath=auto' || '' }} ${{ inputs.free-threading && '--disable-gil' || '' }} - name: Build CPython diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py index cd8e1e84fdcac27..df934164eb9fd60 100644 --- a/Lib/test/test_asyncio/test_tools.py +++ b/Lib/test/test_asyncio/test_tools.py @@ -1,11 +1,9 @@ import unittest -from test.support import import_helper - -_remote_debugging = import_helper.import_module('_remote_debugging') - from asyncio import tools +import _remote_debugging + def LocationInfo(lineno, end_lineno=None, col_offset=None, end_col_offset=None): return _remote_debugging.LocationInfo((lineno, end_lineno, col_offset, end_col_offset)) diff --git a/Modules/_remote_debugging/binary_io_reader.c b/Modules/_remote_debugging/binary_io_reader.c index 9625ee6f301f05f..8af1d281cee6b68 100644 --- a/Modules/_remote_debugging/binary_io_reader.c +++ b/Modules/_remote_debugging/binary_io_reader.c @@ -19,6 +19,10 @@ #include #endif +#ifdef _Py_MEMORY_SANITIZER +# include +#endif + /* ============================================================================ * CONSTANTS FOR BINARY FORMAT SIZES * ============================================================================ */ @@ -315,6 +319,7 @@ reader_decompress_samples(BinaryReader *reader, const uint8_t *data) return -1; } + _Py_MSAN_UNPOISON(output.dst, output.pos); total_output += output.pos; } diff --git a/Modules/_remote_debugging/binary_io_writer.c b/Modules/_remote_debugging/binary_io_writer.c index 6af81515e7131d1..9ea0caa3b82b2b7 100644 --- a/Modules/_remote_debugging/binary_io_writer.c +++ b/Modules/_remote_debugging/binary_io_writer.c @@ -19,6 +19,10 @@ #include #endif +#ifdef _Py_MEMORY_SANITIZER +# include +#endif + /* ============================================================================ * CONSTANTS FOR BINARY FORMAT SIZES * ============================================================================ */ @@ -235,6 +239,7 @@ writer_flush_buffer(BinaryWriter *writer) return -1; } + _Py_MSAN_UNPOISON(writer->zstd.compressed_buffer, output.pos); if (output.pos > 0) { if (fwrite_checked_allow_threads(writer->zstd.compressed_buffer, output.pos, writer->fp) < 0) { return -1; @@ -1084,6 +1089,7 @@ binary_writer_finalize(BinaryWriter *writer) return -1; } + _Py_MSAN_UNPOISON(writer->zstd.compressed_buffer, output.pos); if (output.pos > 0) { if (fwrite_checked_allow_threads(writer->zstd.compressed_buffer, output.pos, writer->fp) < 0) { return -1; diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 806d3fbf5d6b192..25663ce5430d2ee 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1690,6 +1690,7 @@ allocate_instrumentation_data(PyCodeObject *code) } monitoring->local_monitors = (_Py_LocalMonitors){ 0 }; monitoring->active_monitors = (_Py_LocalMonitors){ 0 }; + memset(monitoring->tool_versions, 0, sizeof(monitoring->tool_versions)); monitoring->tools = NULL; monitoring->lines = NULL; monitoring->line_tools = NULL; From 5c0f7ae7f1f94f880d5a593433ccaf6c1cc98d51 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Sat, 3 Oct 2026 15:01:49 +0100 Subject: [PATCH 3/4] Add `_Py_MSAN_UNPOISON_STRING` --- Include/pyport.h | 4 ++++ Modules/posixmodule.c | 4 +--- Modules/socketmodule.c | 8 ++------ 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Include/pyport.h b/Include/pyport.h index 9cfdd09689d5c81..1962338e2719861 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -558,6 +558,7 @@ extern "C" { # define _Py_MEMORY_SANITIZER # define _Py_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory)) # define _Py_MSAN_UNPOISON(PTR, SIZE) (__msan_unpoison(PTR, SIZE)) +# define _Py_MSAN_UNPOISON_STRING(PTR) (__msan_unpoison_string(PTR)) # endif # endif # if __has_feature(address_sanitizer) @@ -599,6 +600,9 @@ extern "C" { #ifndef _Py_MSAN_UNPOISON # define _Py_MSAN_UNPOISON(PTR, SIZE) #endif +#ifndef _Py_MSAN_UNPOISON_STRING +# define _Py_MSAN_UNPOISON_STRING(PTR) +#endif /* AIX has __bool__ redefined in it's system header file. */ #if defined(_AIX) && defined(__bool__) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index e0ac41138b1f605..eacf6556c1ffa41 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -10138,9 +10138,7 @@ os_getlogin_impl(PyObject *module) errno = old_errno; } else { -#ifdef _Py_MEMORY_SANITIZER - __msan_unpoison(name, sizeof(name)); -#endif + _Py_MSAN_UNPOISON(name, sizeof(name)); result = PyUnicode_DecodeFSDefault(name); } #else diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index bde7728634d7b8c..f71ed9801e5a225 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -763,9 +763,7 @@ set_error(void) static PyObject * decode_error_message(const char *str) { -#ifdef _Py_MEMORY_SANITIZER - __msan_unpoison_string(str); -#endif + _Py_MSAN_UNPOISON_STRING(str); return PyUnicode_DecodeLocale(str, "surrogateescape"); } #endif @@ -6525,9 +6523,7 @@ _socket_getservbyport_impl(PyObject *module, int port, const char *proto) PyErr_SetString(PyExc_OSError, "port/proto not found"); return NULL; } -#ifdef _Py_MEMORY_SANITIZER - __msan_unpoison_string(sp->s_name); -#endif + _Py_MSAN_UNPOISON_STRING(sp->s_name); return PyUnicode_FromString(sp->s_name); } From ab2ad7e0c870e364242e3d377402db29cfcc31e2 Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Sat, 3 Oct 2026 17:11:30 +0100 Subject: [PATCH 4/4] Apply Victor's suggestions Co-authored-by: Victor Stinner --- Include/pyport.h | 4 ++-- Lib/test/test_faulthandler.py | 4 ++-- Modules/socketmodule.c | 9 ++++++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Include/pyport.h b/Include/pyport.h index 1962338e2719861..2206beaa77221c2 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -558,7 +558,7 @@ extern "C" { # define _Py_MEMORY_SANITIZER # define _Py_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory)) # define _Py_MSAN_UNPOISON(PTR, SIZE) (__msan_unpoison(PTR, SIZE)) -# define _Py_MSAN_UNPOISON_STRING(PTR) (__msan_unpoison_string(PTR)) +# define _Py_MSAN_UNPOISON_STRING(STR) (__msan_unpoison_string(STR)) # endif # endif # if __has_feature(address_sanitizer) @@ -601,7 +601,7 @@ extern "C" { # define _Py_MSAN_UNPOISON(PTR, SIZE) #endif #ifndef _Py_MSAN_UNPOISON_STRING -# define _Py_MSAN_UNPOISON_STRING(PTR) +# define _Py_MSAN_UNPOISON_STRING(STR) #endif /* AIX has __bool__ redefined in it's system header file. */ diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 5a493a4fd956802..82b347c8f8c045b 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -34,8 +34,8 @@ def skip_if_sanitizer_signal(signame): - return support.skip_if_sanitizer(f"TSAN/UBSan itercepts {signame}", - thread=True, ub=True) + return support.skip_if_sanitizer(f"TSan/UBSan/MSan intercepts {signame}", + thread=True, ub=True, memory=True) def expected_traceback(lineno1, lineno2, header, min_count=1): diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index f71ed9801e5a225..5fec77a2b82bf7e 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -763,7 +763,6 @@ set_error(void) static PyObject * decode_error_message(const char *str) { - _Py_MSAN_UNPOISON_STRING(str); return PyUnicode_DecodeLocale(str, "surrogateescape"); } #endif @@ -775,7 +774,9 @@ set_herror(socket_state *state, int h_error) PyObject *v; #ifdef HAVE_HSTRERROR - v = Py_BuildValue("(iN)", h_error, decode_error_message(hstrerror(h_error))); + const char *errmsg = hstrerror(h_error); + _Py_MSAN_UNPOISON_STRING(errmsg); + v = Py_BuildValue("(iN)", h_error, decode_error_message(errmsg)); #else v = Py_BuildValue("(is)", h_error, "host not found"); #endif @@ -802,7 +803,9 @@ set_gaierror(socket_state *state, int error) #endif #ifdef HAVE_GAI_STRERROR - v = Py_BuildValue("(iN)", error, decode_error_message(gai_strerror(error))); + const char *errmsg = gai_strerror(error); + _Py_MSAN_UNPOISON_STRING(errmsg); + v = Py_BuildValue("(iN)", error, decode_error_message(errmsg)); #else v = Py_BuildValue("(is)", error, "getaddrinfo failed"); #endif