From 48f1249d23831cdd341d5a790b64c8599e5a69b0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 2 Oct 2026 20:54:30 +0200 Subject: [PATCH 1/2] gh-158585: Adjust allocation in bytes_fromformat() Take in account preallocated bytes when calling PyBytesWriter_GrowAndUpdatePointer(). Use sprintf() return value instead of calling strlen(). --- Lib/test/test_capi/test_bytes.py | 32 ++++++++- Objects/bytesobject.c | 119 +++++++++++++++++-------------- 2 files changed, 95 insertions(+), 56 deletions(-) diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index f8487a2b8fb085..910f2bedac0bcb 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -607,15 +607,45 @@ def test_format_i(self): writer = self.create_writer() writer.format_i(b'x=%i, ', 123) - writer.format_i(b'y=%i', 456) + writer.format_i(b'y=%d', 456) self.bytes_equal(writer.finish(), b'x=123, y=456') + # Make sure that %i allocates the right number of bytes + n_str = '1234567' + for i in range(1, len(n_str) + 1): + n = int(n_str[:i]) + writer = self.create_writer(0) + writer.format_i(b'n=%i', n) + self.bytes_equal(writer.finish(), f'n={n}'.encode()) + + # Test invalid format: "%t" is invalid, stop the parser and copy + # remaining bytes + writer = self.create_writer() + writer.write_bytes(b'prefix ', -1) + writer.format_i(b'y=%._ t %i', 0) + writer.write_bytes(b'.', 1) + self.bytes_equal(writer.finish(), b'prefix y=%._ t %i.') + def test_format_s(self): # Test PyBytesWriter_Format() writer = self.create_writer() writer.format_s(b's=%s', b'Hello World') self.bytes_equal(writer.finish(), b's=Hello World') + # Test precision + writer = self.create_writer() + writer.format_s(b'%.6s', b'Monty Python') + writer.format_s(b'%.20s', b'3.0') + self.bytes_equal(writer.finish(), b'Monty 3.0') + + # Make sure that %s allocates the right number of bytes + text = b'1234567' + for i in range(1, len(text) + 1): + s = text[:i] + writer = self.create_writer(0) + writer.format_s(b's=%s', s) + self.bytes_equal(writer.finish(), b's=' + s) + @support.nomemtest def test_format_s_memory_error(self): writer = self.create_writer() diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index cb2c2f7bfae634..9c0264b6d49b06 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -194,73 +194,77 @@ static char* bytes_fromformat(PyBytesWriter *writer, Py_ssize_t writer_pos, const char *format, va_list vargs) { - const char *f; - const char *p; - Py_ssize_t prec; - int longflag; - int size_tflag; /* Longest 64-bit formatted numbers: - "18446744073709551615\0" (21 bytes) - "-9223372036854775808\0" (21 bytes) Decimal takes the most space (it isn't enough for octal.) Longest 64-bit pointer representation: - "0xffffffffffffffff\0" (19 bytes). */ + "0xffffffffffffffff\0" (19 bytes). + + Longest 64-bit "%p" with "0x" prefix: len(hex(2**64-1)+'\0') = 19. + */ char buffer[21]; char *s = (char*)PyBytesWriter_GetData(writer) + writer_pos; -#define WRITE_BYTES_LEN(str, len_expr) \ +#define WRITE_BYTES(str, len_expr) \ do { \ - size_t len = (len_expr); \ - s = PyBytesWriter_GrowAndUpdatePointer(writer, len, s); \ - if (s == NULL) { \ - return NULL; \ + size_t _len = (len_expr); \ + size_t _prealloc = (f - p + 1); \ + if (_len > _prealloc) { \ + s = PyBytesWriter_GrowAndUpdatePointer(writer, _len - _prealloc, s); \ + if (s == NULL) { \ + return NULL; \ + } \ } \ - memcpy(s, (str), len); \ - s += len; \ + memcpy(s, (str), _len); \ + s += _len; \ } while (0) -#define WRITE_BYTES(str) WRITE_BYTES_LEN(str, strlen(str)) - for (f = format; *f; f++) { + for (const char *f = format; *f; f++) { if (*f != '%') { *s++ = *f; continue; } - p = f++; + const char *p = f++; /* ignore the width (ex: 10 in "%10s") */ while (Py_ISDIGIT(*f)) f++; /* parse the precision (ex: 10 in "%.10s") */ - prec = 0; + Py_ssize_t prec = 0; if (*f == '.') { f++; for (; Py_ISDIGIT(*f); f++) { prec = (prec * 10) + (*f - '0'); } } + assert(prec >= 0); - while (*f && *f != '%' && !Py_ISALPHA(*f)) + while (*f && *f != '%' && !Py_ISALPHA(*f)) { f++; + } /* handle the long flag ('l'), but only for %ld and %lu. others can be added when necessary. */ - longflag = 0; + int longflag = 0; if (*f == 'l' && (f[1] == 'd' || f[1] == 'u')) { longflag = 1; ++f; } /* handle the size_t flag ('z'). */ - size_tflag = 0; + int size_tflag = 0; if (*f == 'z' && (f[1] == 'd' || f[1] == 'u')) { size_tflag = 1; ++f; } + Py_ssize_t len; + switch (*f) { case 'c': { @@ -277,74 +281,76 @@ bytes_fromformat(PyBytesWriter *writer, Py_ssize_t writer_pos, case 'd': if (longflag) { - sprintf(buffer, "%ld", va_arg(vargs, long)); + len = sprintf(buffer, "%ld", va_arg(vargs, long)); } else if (size_tflag) { - sprintf(buffer, "%zd", va_arg(vargs, Py_ssize_t)); + len = sprintf(buffer, "%zd", va_arg(vargs, Py_ssize_t)); } else { - sprintf(buffer, "%d", va_arg(vargs, int)); + len = sprintf(buffer, "%d", va_arg(vargs, int)); } - assert(strlen(buffer) < sizeof(buffer)); - WRITE_BYTES(buffer); + assert(1 <= len && len < (Py_ssize_t)sizeof(buffer)); + WRITE_BYTES(buffer, len); break; case 'u': if (longflag) { - sprintf(buffer, "%lu", va_arg(vargs, unsigned long)); + len = sprintf(buffer, "%lu", va_arg(vargs, unsigned long)); } else if (size_tflag) { - sprintf(buffer, "%zu", va_arg(vargs, size_t)); + len = sprintf(buffer, "%zu", va_arg(vargs, size_t)); } else { - sprintf(buffer, "%u", va_arg(vargs, unsigned int)); + len = sprintf(buffer, "%u", va_arg(vargs, unsigned int)); } - assert(strlen(buffer) < sizeof(buffer)); - WRITE_BYTES(buffer); + assert(1 <= len && len < (Py_ssize_t)sizeof(buffer)); + WRITE_BYTES(buffer, len); break; case 'i': - sprintf(buffer, "%i", va_arg(vargs, int)); - assert(strlen(buffer) < sizeof(buffer)); - WRITE_BYTES(buffer); + len = sprintf(buffer, "%i", va_arg(vargs, int)); + assert(1 <= len && len < (Py_ssize_t)sizeof(buffer)); + WRITE_BYTES(buffer, len); break; case 'x': - sprintf(buffer, "%x", va_arg(vargs, int)); - assert(strlen(buffer) < sizeof(buffer)); - WRITE_BYTES(buffer); + len = sprintf(buffer, "%x", va_arg(vargs, int)); + assert(1 <= len && len < (Py_ssize_t)sizeof(buffer)); + WRITE_BYTES(buffer, len); break; case 's': { - Py_ssize_t i; - - p = va_arg(vargs, const char*); - if (prec <= 0) { - i = strlen(p); + const char *str = va_arg(vargs, const char*); + if (prec == 0) { + len = strlen(str); } else { - i = 0; - while (i < prec && p[i]) { - i++; + const char *end = memchr(str, 0, prec); + if (end != NULL) { + len = (end - str); + } + else { + len = (size_t)prec; } } - WRITE_BYTES_LEN(p, i); + WRITE_BYTES(str, len); break; } case 'p': - sprintf(buffer, "%p", va_arg(vargs, void*)); - assert(strlen(buffer) < sizeof(buffer)); + len = sprintf(buffer, "%p", va_arg(vargs, void*)); + assert(1 <= len && len < (Py_ssize_t)sizeof(buffer)); /* %p is ill-defined: ensure leading 0x. */ - if (buffer[1] == 'X') + if (buffer[1] == 'X') { buffer[1] = 'x'; + } else if (buffer[1] != 'x') { - memmove(buffer+2, buffer, strlen(buffer)+1); + memmove(buffer + 2, buffer, len + 1); buffer[0] = '0'; buffer[1] = 'x'; } - WRITE_BYTES(buffer); + WRITE_BYTES(buffer, len); break; case '%': @@ -352,16 +358,19 @@ bytes_fromformat(PyBytesWriter *writer, Py_ssize_t writer_pos, break; default: - /* invalid format string: copy unformatted string and exit */ - WRITE_BYTES(p); + // Invalid format string: copy unformatted string and exit. + // No need to grow the writer buffer, we already preallocated + // enough bytes. + len = strlen(p); + memcpy(s, p, len); + s += len; return s; } } -#undef WRITE_BYTES -#undef WRITE_BYTES_LEN - return s; + +#undef WRITE_BYTES } From 4ab5fefe3be7e535e17314cf8e729ef1644f6432 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 2 Oct 2026 23:04:55 +0200 Subject: [PATCH 2/2] Fix %p format on Windows/macOS --- Objects/bytesobject.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 9c0264b6d49b06..2339e8bc1d2819 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -349,6 +349,7 @@ bytes_fromformat(PyBytesWriter *writer, Py_ssize_t writer_pos, memmove(buffer + 2, buffer, len + 1); buffer[0] = '0'; buffer[1] = 'x'; + len += 2; } WRITE_BYTES(buffer, len); break;