diff --git a/Include/internal/pycore_freelist_state.h b/Include/internal/pycore_freelist_state.h index 46e2a82ea03456..95c59d9b5cde35 100644 --- a/Include/internal/pycore_freelist_state.h +++ b/Include/internal/pycore_freelist_state.h @@ -9,6 +9,7 @@ extern "C" { #endif # define PyTuple_MAXSAVESIZE 20 // Largest tuple to save on freelist +# define PyBytes_FREELIST_BUCKETS 8 # define Py_tuple_MAXFREELIST 2000 // Maximum number of tuples of each size to save # define Py_lists_MAXFREELIST 80 # define Py_list_iters_MAXFREELIST 10 @@ -28,6 +29,7 @@ extern "C" { # define Py_object_stack_chunks_MAXFREELIST 4 # define Py_unicode_writers_MAXFREELIST 1 # define Py_bytes_writers_MAXFREELIST 1 +# define Py_bytes_MAXFREELIST 20 # define Py_pycfunctionobject_MAXFREELIST 16 # define Py_pycmethodobject_MAXFREELIST 16 # define Py_pymethodobjects_MAXFREELIST 20 @@ -63,6 +65,7 @@ struct _Py_freelists { struct _Py_freelist object_stack_chunks; struct _Py_freelist unicode_writers; struct _Py_freelist bytes_writers; + struct _Py_freelist bytes[PyBytes_FREELIST_BUCKETS]; struct _Py_freelist pycfunctionobject; struct _Py_freelist pycmethodobject; struct _Py_freelist pymethodobjects; diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 2339e8bc1d2819..5c2e93fe639b99 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -28,6 +28,19 @@ class bytes "PyBytesObject *" "&PyBytes_Type" #include "clinic/bytesobject.c.h" #define PyBytesObject_SIZE _PyBytesObject_SIZE +#define FREELIST_MAX_SIZE 255 +#define PyBytes_FREELIST_BUCKET(size) ((size) / 32) + +static inline Py_ssize_t +PyBytes_FREELIST_ALLOC(Py_ssize_t size) +{ + if (size <= FREELIST_MAX_SIZE) { + return (PyBytes_FREELIST_BUCKET(size) * 32 + 31); + } + else { + return size; + } +} /* Forward declaration */ static void* _PyBytesWriter_ResizeAndUpdatePointer(PyBytesWriter *writer, @@ -116,10 +129,28 @@ _PyBytes_FromSize(Py_ssize_t size, int use_calloc) } /* Inline PyObject_NewVar */ - if (use_calloc) - op = (PyBytesObject *)PyObject_Calloc(1, PyBytesObject_SIZE + size); - else - op = (PyBytesObject *)PyObject_Malloc(PyBytesObject_SIZE + size); + if (size <= FREELIST_MAX_SIZE) { + Py_ssize_t alloc = PyBytes_FREELIST_ALLOC(size); + Py_ssize_t bucket = PyBytes_FREELIST_BUCKET(size); + op = _Py_FREELIST_POP_MEM(bytes[bucket]); + if (op != NULL) { + if (use_calloc) { + memset(op, 0, PyBytesObject_SIZE + alloc); + } + } + else { + if (use_calloc) + op = (PyBytesObject *)PyObject_Calloc(1, PyBytesObject_SIZE + alloc); + else + op = (PyBytesObject *)PyObject_Malloc(PyBytesObject_SIZE + alloc); + } + } + else { + if (use_calloc) + op = (PyBytesObject *)PyObject_Calloc(1, PyBytesObject_SIZE + size); + else + op = (PyBytesObject *)PyObject_Malloc(PyBytesObject_SIZE + size); + } if (op == NULL) { return PyErr_NoMemory(); } @@ -165,11 +196,6 @@ PyBytes_FromString(const char *str) assert(str != NULL); size = strlen(str); - if (size > PY_SSIZE_T_MAX - PyBytesObject_SIZE) { - PyErr_SetString(PyExc_OverflowError, - "byte string is too long"); - return NULL; - } if (size == 0) { return bytes_get_empty(); @@ -179,12 +205,10 @@ PyBytes_FromString(const char *str) } /* Inline PyObject_NewVar */ - op = (PyBytesObject *)PyObject_Malloc(PyBytesObject_SIZE + size); + op = (PyBytesObject *)_PyBytes_FromSize(size, 0); if (op == NULL) { return PyErr_NoMemory(); } - _PyObject_InitVar((PyVarObject*)op, &PyBytes_Type, size); - set_ob_shash(op, -1); memcpy(op->ob_sval, str, size+1); return (PyObject *) op; } @@ -1604,19 +1628,10 @@ _PyBytes_Repeat(PyObject *self, Py_ssize_t n) if (size == Py_SIZE(a) && PyBytes_CheckExact(a)) { return Py_NewRef(a); } - size_t nbytes = (size_t)size; - if (nbytes + PyBytesObject_SIZE <= nbytes) { - PyErr_SetString(PyExc_OverflowError, - "repeated bytes are too long"); - return NULL; - } - PyBytesObject *op = PyObject_Malloc(PyBytesObject_SIZE + nbytes); + PyBytesObject *op = (PyBytesObject *)_PyBytes_FromSize(size, 0); if (op == NULL) { return PyErr_NoMemory(); } - _PyObject_InitVar((PyVarObject*)op, &PyBytes_Type, size); - set_ob_shash(op, -1); - op->ob_sval[size] = '\0'; _PyBytes_RepeatBuffer(op->ob_sval, size, a->ob_sval, Py_SIZE(a)); @@ -3212,16 +3227,28 @@ _PyBytes_CheckOverflow(PyObject *self, void *addr, const char *type_name) type_name, addr, size); } } +#endif static void bytes_dealloc(PyObject *op) { PyBytesObject *self = _PyBytes_CAST(op); + PyTypeObject *type = Py_TYPE(self); + Py_ssize_t size = PyBytes_GET_SIZE(self); +#ifdef Py_DEBUG _PyBytes_CheckOverflow(op, op, "bytes"); - Py_TYPE(self)->tp_free((PyObject *)self); -} #endif + if (size <= FREELIST_MAX_SIZE && type == &PyBytes_Type) { + Py_ssize_t bucket = PyBytes_FREELIST_BUCKET(size); + if (!_Py_FREELIST_PUSH(bytes[bucket], op, Py_bytes_MAXFREELIST)) { + PyObject_Free(op); + } + } + else { + type->tp_free((PyObject *)self); + } +} PyTypeObject PyBytes_Type = { @@ -3229,11 +3256,7 @@ PyTypeObject PyBytes_Type = { "bytes", PyBytesObject_SIZE, sizeof(char), -#ifdef Py_DEBUG bytes_dealloc, /* tp_dealloc */ -#else - 0, /* tp_dealloc */ -#endif 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -3377,7 +3400,9 @@ bytes_resize_inplace(PyObject **pv, Py_ssize_t newsize) // Only mutable bytes can be resized in-place assert(_PyBytes_IsMutable(v)); - if ((size_t)newsize > (size_t)PY_SSIZE_T_MAX - PyBytesObject_SIZE) { +#define FREELIST_MAX_SIZE 255 + Py_ssize_t alloc = PyBytes_FREELIST_ALLOC(newsize); + if ((size_t)alloc > (size_t)PY_SSIZE_T_MAX - PyBytesObject_SIZE) { PyErr_SetString(PyExc_OverflowError, "byte string is too large"); return -1; @@ -3388,7 +3413,7 @@ bytes_resize_inplace(PyObject **pv, Py_ssize_t newsize) #endif _PyReftracerTrack(v, PyRefTracer_DESTROY); - PyObject *result = PyObject_Realloc(v, PyBytesObject_SIZE + newsize); + PyObject *result = PyObject_Realloc(v, PyBytesObject_SIZE + alloc); if (result == NULL) { #ifdef Py_TRACE_REFS _Py_AddToAllObjects(v); diff --git a/Objects/object.c b/Objects/object.c index c7aeba0cee22c6..b11ed43b0d24c2 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -952,6 +952,9 @@ _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization) } clear_freelist(&freelists->unicode_writers, is_finalization, PyMem_Free); clear_freelist(&freelists->bytes_writers, is_finalization, PyMem_Free); + for (Py_ssize_t i = 0; i < PyBytes_FREELIST_BUCKETS; i++) { + clear_freelist(&freelists->bytes[i], is_finalization, PyObject_Free); + } clear_freelist(&freelists->ints, is_finalization, free_object); clear_freelist(&freelists->pycfunctionobject, is_finalization, PyObject_GC_Del); clear_freelist(&freelists->pycmethodobject, is_finalization, PyObject_GC_Del);