Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions Lib/test/pickletester.py
Original file line number Diff line number Diff line change
Expand Up @@ -4925,6 +4925,31 @@ def test_load_from_and_dump_to_file(self):
unpickled = self.load(stream)
self.assertEqual(unpickled, data)

def test_load_from_file_returning_bytes_subclass(self):
# gh-158841: read() returning a bytes subclass must not crash
# when the payload is large enough to need several chunked reads.
class MyBytes(bytes):
pass

class Reader:
def __init__(self, data):
self.data = data
self.pos = 0
def read(self, n):
chunk = self.data[self.pos:self.pos + n]
self.pos += len(chunk)
return MyBytes(chunk)
def readline(self):
end = self.data.find(b'\n', self.pos)
end = len(self.data) if end < 0 else end + 1
line = self.data[self.pos:end]
self.pos = end
return line

obj = {'v': 'B' * (3 * 1024 * 1024)}
data = self.dumps(obj, protocol=4)
self.assertEqual(self.load(Reader(data)), obj)

def test_highest_protocol(self):
# Of course this needs to be changed when HIGHEST_PROTOCOL changes.
self.assertEqual(pickle.HIGHEST_PROTOCOL, 5)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Fix a crash in :func:`pickle.load` and :class:`pickle.Unpickler` when the
``read()`` method of the file object returns an instance of a :class:`bytes`
subclass and the requested size is large.
11 changes: 11 additions & 0 deletions Modules/_pickle.c
Original file line number Diff line number Diff line change
Expand Up @@ -1439,6 +1439,17 @@ _Unpickler_ReadFromFile(PickleState *state, UnpicklerObject *self, Py_ssize_t n)
if (data == NULL) {
return -1;
}
if (cursize < n && PyBytes_Check(data) && !PyBytes_CheckExact(data)) {
/* read() may return a bytes subclass, which cannot be resized
in place. Copy it into an exact bytes object. */
PyObject *exact = PyBytes_FromStringAndSize(
PyBytes_AS_STRING(data), PyBytes_GET_SIZE(data));
Py_DECREF(data);
if (exact == NULL) {
return -1;
}
data = exact;
}
while (cursize < n) {
Py_ssize_t prevsize = cursize;
// geometrically double the chunk size to avoid CPU DoS
Expand Down