From 920cc2b239ca719f4cd20c1377b0c30913ce3ed7 Mon Sep 17 00:00:00 2001 From: jg Date: Wed, 30 Sep 2026 08:47:21 +0100 Subject: [PATCH 1/3] Avoid second buffer fetch on bytearray_decode --- Lib/test/test_bytes.py | 24 +++++++++++++++++++ ...-09-30-10-12-41.gh-issue-158316.Yv3kQe.rst | 2 ++ Objects/bytearrayobject.c | 14 ++++++++++- 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-10-12-41.gh-issue-158316.Yv3kQe.rst diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index b55863256cc37c..b923f31178cc62 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -2195,6 +2195,30 @@ def delslice(): self.assertRaises(BufferError, delslice) self.assertEqual(b, orig) + def test_decode_resize_forbidden(self): + # The storage is pinned while it is decoded, so an error handler + # cannot resize the bytearray. + b = bytearray(b'ab\xffcd') + def handler(exc): + self.assertRaises(BufferError, b.clear) + self.assertRaises(BufferError, b.append, 0) + return ('?', exc.end) + codecs.register_error('test.bytearray_decode_resize', handler) + for encoding in 'utf-8', 'utf-8-sig': + with self.subTest(encoding=encoding): + self.assertEqual( + b.decode(encoding, 'test.bytearray_decode_resize'), + 'ab?cd') + self.assertEqual(b, b'ab\xffcd') + + def test_decode_subclass_buffer(self): + # decode() decodes the buffer that the object exports. + class B(bytearray): + def __buffer__(self, flags): + return memoryview(b'other') + self.assertEqual(B(b'mine').decode(), 'other') + self.assertEqual(B(b'mine').decode('latin-1'), 'other') + @test.support.cpython_only def test_obsolete_write_lock(self): _testcapi = import_helper.import_module('_testcapi') diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-10-12-41.gh-issue-158316.Yv3kQe.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-10-12-41.gh-issue-158316.Yv3kQe.rst new file mode 100644 index 00000000000000..04b59649f01f55 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-10-12-41.gh-issue-158316.Yv3kQe.rst @@ -0,0 +1,2 @@ +Speed up :meth:`bytearray.decode` by decoding the underlying storage directly +instead of going through the buffer protocol. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index caf8bad88a280d..496d04a1704d46 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2599,7 +2599,19 @@ bytearray_decode_impl(PyByteArrayObject *self, const char *encoding, { if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); - return PyUnicode_FromEncodedObject((PyObject*)self, encoding, errors); + if (Py_TYPE(self)->tp_as_buffer->bf_getbuffer != bytearray_getbuffer) { + /* A subclass may export a different buffer. */ + return PyUnicode_FromEncodedObject((PyObject*)self, encoding, errors); + } + + /* Decode the storage directly instead of exporting a buffer, which + would re-acquire the critical section we already hold. Increase + exports to prevent the storage from changing during the decode. */ + self->ob_exports++; + PyObject *res = PyUnicode_Decode(PyByteArray_AS_STRING(self), + Py_SIZE(self), encoding, errors); + self->ob_exports--; + return res; } PyDoc_STRVAR(alloc_doc, From 04317fc564a54d6dd606e85122f016d95ac54605 Mon Sep 17 00:00:00 2001 From: jg Date: Wed, 30 Sep 2026 18:38:41 +0100 Subject: [PATCH 2/3] PR comments on the resize test --- Lib/test/test_bytes.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index b923f31178cc62..f09cd2d45eb442 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -18,6 +18,7 @@ import textwrap import threading import unittest +from _codecs import _unregister_error as _codecs_unregister_error import test.support from test import support @@ -2203,13 +2204,14 @@ def handler(exc): self.assertRaises(BufferError, b.clear) self.assertRaises(BufferError, b.append, 0) return ('?', exc.end) + self.addCleanup(_codecs_unregister_error, 'test.bytearray_decode_resize') codecs.register_error('test.bytearray_decode_resize', handler) for encoding in 'utf-8', 'utf-8-sig': with self.subTest(encoding=encoding): self.assertEqual( b.decode(encoding, 'test.bytearray_decode_resize'), 'ab?cd') - self.assertEqual(b, b'ab\xffcd') + self.assertEqual(b, b'ab\xffcd') def test_decode_subclass_buffer(self): # decode() decodes the buffer that the object exports. From aab90715bb289f3f6f595234c04d80dc6dbf4606 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 1 Oct 2026 21:45:30 +0200 Subject: [PATCH 3/3] Apply batched suggestions from code review Co-authored-by: Victor Stinner --- Lib/test/test_bytes.py | 9 ++++----- .../2026-09-30-10-12-41.gh-issue-158316.Yv3kQe.rst | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index f09cd2d45eb442..ea0a54dd5e1d35 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -2200,17 +2200,16 @@ def test_decode_resize_forbidden(self): # The storage is pinned while it is decoded, so an error handler # cannot resize the bytearray. b = bytearray(b'ab\xffcd') + errors = 'test.bytearray_decode_resize' def handler(exc): self.assertRaises(BufferError, b.clear) self.assertRaises(BufferError, b.append, 0) return ('?', exc.end) - self.addCleanup(_codecs_unregister_error, 'test.bytearray_decode_resize') - codecs.register_error('test.bytearray_decode_resize', handler) + self.addCleanup(_codecs_unregister_error, errors) + codecs.register_error(errors, handler) for encoding in 'utf-8', 'utf-8-sig': with self.subTest(encoding=encoding): - self.assertEqual( - b.decode(encoding, 'test.bytearray_decode_resize'), - 'ab?cd') + self.assertEqual(b.decode(encoding, errors), 'ab?cd') self.assertEqual(b, b'ab\xffcd') def test_decode_subclass_buffer(self): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-10-12-41.gh-issue-158316.Yv3kQe.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-10-12-41.gh-issue-158316.Yv3kQe.rst index 04b59649f01f55..c95bab19af393a 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-10-12-41.gh-issue-158316.Yv3kQe.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-30-10-12-41.gh-issue-158316.Yv3kQe.rst @@ -1,2 +1,2 @@ -Speed up :meth:`bytearray.decode` by decoding the underlying storage directly -instead of going through the buffer protocol. +On Free Threading, speed up :meth:`bytearray.decode` by decoding the underlying +storage directly instead of going through the buffer protocol.