Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions Lib/test/test_bytes.py
Original file line number Diff line number Diff line change
Expand Up @@ -2197,6 +2197,50 @@ def __len__(self):

self.assertRaises(BufferError, ba.hex, S(b':'))

def test_uninitialized_instance(self):
# A bytearray created with __new__ so that __init__ is never called
# (often as a side-effect of a subclass not calling super().__init__)
# is left with ob_bytes_object == NULL. It's easy for implementation
# code to not realize that ob_bytes_object can be NULL, so these
# checks exercise code paths that have historically crashed or
# asserted (see gh-153419).
def uninitialized():
return bytearray.__new__(bytearray)

uninitialized().insert(0, 1)
uninitialized().extend(b"x")
uninitialized().extend([1, 2, 3])
uninitialized().resize(4)
uninitialized().__init__(5)
uninitialized().__init__(b"xyz")
uninitialized().take_bytes()
uninitialized().take_bytes(0)

a = uninitialized()
a.append(1)

a = uninitialized()
a += b"x"

a = uninitialized()
a[:] = b"xyz"

def test_reinit_length1(self):
# There is a shortcut taken when resizing, where alloc/2 < newsize.
# In this case, the existing buffer is reused, rather than reset.
# If this happens when newsize == 0 and alloc == 1, then various
# code assumptions can be violated. This test should catch those
# in debug builds. (see gh-153419)
a = bytearray(1)
a.__init__()
self.assertEqual(a, b"")

def test_reinit_with_view(self):
a = bytearray()
with memoryview(a):
self.assertRaises(BufferError, a.__init__, "x", "ascii")
self.assertEqual(a, b"")


class AssortedBytesTest(unittest.TestCase):
#
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fix several :class:`bytearray` crashes caused by calling :meth:`~object.__init__`/:meth:`~object.__new__` in unusual ways.
39 changes: 29 additions & 10 deletions Objects/bytearrayobject.c
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,15 @@ bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size)
{
_Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self);
PyByteArrayObject *obj = ((PyByteArrayObject *)self);

/* If ob_bytes_object has not been initialized yet, eagerly initialize
it here so the following code can reason about state more easily,
and things like pointer comparisons are valid. */
if (obj->ob_bytes_object == NULL) {
obj->ob_bytes_object = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES);
bytearray_reinit_from_bytes(obj, 0, 0);
}

/* All computations are done unsigned to avoid integer overflows
(see issue #22335). */
size_t alloc = (size_t) obj->ob_alloc;
Expand All @@ -236,6 +245,15 @@ bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size)
return -1;
}

/* When resizing to 0, always reset to the empty-bytes constant to avoid
complexity in the realloc path below (see gh-153419). */
if (requested_size == 0) {
Py_XSETREF(obj->ob_bytes_object,
Py_GetConstant(Py_CONSTANT_EMPTY_BYTES));
bytearray_reinit_from_bytes(obj, 0, 0);
return 0;
}

if (size + logical_offset <= alloc) {
/* Current buffer is large enough to host the requested size,
decide on a strategy. */
Expand Down Expand Up @@ -920,20 +938,17 @@ bytearray___init___impl(PyByteArrayObject *self, PyObject *arg,
PyObject *it;
PyObject *(*iternext)(PyObject *);

/* First __init__; set ob_bytes_object so ob_bytes is always non-null. */

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please keep the NULL check here rather than moving it inside the resize code. This as written I suspect will also fail some of the Sanitizers, ob_exports is unset until the NULL code sets it.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The issue I have with this is that it's kinda misleading and redundant. There's 0 guarantee that this function gets called, so it looks like it's a reliable initialiser without actually being so. I can put back the code here, but we'd have to keep it in resize too, as it's easy to hit the resize call without having init called.

This is partly why I went a bit comment heavy, because reasoning about this is quite hard.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I ran asan as you suggested , and it all checks out, I think the thing is either we accept that it's valid for ob_bytes_object to be NULL sometimes, and just handle that correctly when needed, or we have to make ob_bytes_object never be null, and the only way to do that is with tp_new. (without tricks like PyObject_New which the docs say to avoid using)

As for ob_export, it is zero initialized by default, if it's non-zero then we can't call init anyway, and you get the exception, so I can't see how it would ever need to be reinitialized here. This is different from ..StringAndSize, where the object is allocated with PyObject_New which doesn't zero-initialize the struct.

if (self->ob_bytes_object == NULL) {
self->ob_bytes_object = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES);
bytearray_reinit_from_bytes(self, 0, 0);
self->ob_exports = 0;
if (!_canresize(self)) {
return -1;
}

if (Py_SIZE(self) != 0) {
/* Empty previous contents (yes, do this first of all!) */
if (PyByteArray_Resize((PyObject *)self, 0) < 0)
return -1;
/* Empty any previous contents (do this first of all!).
Also initializes ob_bytes_object if needed */
if (PyByteArray_Resize((PyObject *)self, 0) < 0) {
return -1;
}

/* Should be caused by first init or the resize to 0. */
/* PyByteArray_Resize(,0) should always leave the empty bytes constant */
assert(self->ob_bytes_object == Py_GetConstantBorrowed(Py_CONSTANT_EMPTY_BYTES));
assert(self->ob_exports == 0);

Expand Down Expand Up @@ -1607,6 +1622,10 @@ bytearray_take_bytes_impl(PyByteArrayObject *self, PyObject *n)
}

if (_PyBytes_Resize(&self->ob_bytes_object, to_take) == -1) {
/* _PyBytes_Resize has made ob_bytes_object NULL here. We need to
ensure that the other bytearray fields are consistent. */
self->ob_bytes_object = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES);
bytearray_reinit_from_bytes(self, 0, 0);
Py_DECREF(remaining);
return NULL;
}
Expand Down
Loading