diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 97f934727a85a36..5f66c2a6868d3bf 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -199,6 +199,7 @@ enum { _GUARD_TOS_INT, _GUARD_TOS_IS_NONE, _GUARD_TOS_LIST, + _GUARD_TOS_NON_NEGATIVE_COMPACT_INT, _GUARD_TOS_NOT_NULL, _GUARD_TOS_OVERFLOWED, _GUARD_TOS_SET, @@ -364,7 +365,7 @@ enum { _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, _YIELD_VALUE, }; -#define MAX_UOP_ID 652 +#define MAX_UOP_ID 653 #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST @@ -950,6 +951,10 @@ enum { _GUARD_TOS_LIST_r11, _GUARD_TOS_LIST_r22, _GUARD_TOS_LIST_r33, + _GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r01, + _GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r11, + _GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r22, + _GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r33, _GUARD_TOS_NOT_NULL_r01, _GUARD_TOS_NOT_NULL_r11, _GUARD_TOS_NOT_NULL_r22, @@ -1434,7 +1439,7 @@ enum { _WITH_EXCEPT_START_r33, _YIELD_VALUE_r11, }; -#define MAX_UOP_REGS_ID 1645 +#define MAX_UOP_REGS_ID 1650 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 990706c0b329223..2c594c620d36519 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -156,6 +156,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LIST_APPEND] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_SET_ADD] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_TOS_NON_NEGATIVE_COMPACT_INT] = HAS_EXIT_FLAG, [_STORE_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_DICT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SUBSCR_DICT_KNOWN_HASH] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -1533,6 +1534,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 0, 3, _STORE_SUBSCR_r30 }, }, }, + [_GUARD_TOS_NON_NEGATIVE_COMPACT_INT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r01 }, + { 1, 1, _GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r11 }, + { 2, 2, _GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r22 }, + { 3, 3, _GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r33 }, + }, + }, [_STORE_SUBSCR_LIST_INT] = { .best = { 3, 3, 3, 3 }, .entries = { @@ -4237,6 +4247,10 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_LIST_APPEND_r10] = _LIST_APPEND, [_SET_ADD_r10] = _SET_ADD, [_STORE_SUBSCR_r30] = _STORE_SUBSCR, + [_GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r01] = _GUARD_TOS_NON_NEGATIVE_COMPACT_INT, + [_GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r11] = _GUARD_TOS_NON_NEGATIVE_COMPACT_INT, + [_GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r22] = _GUARD_TOS_NON_NEGATIVE_COMPACT_INT, + [_GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r33] = _GUARD_TOS_NON_NEGATIVE_COMPACT_INT, [_STORE_SUBSCR_LIST_INT_r32] = _STORE_SUBSCR_LIST_INT, [_STORE_SUBSCR_DICT_r31] = _STORE_SUBSCR_DICT, [_STORE_SUBSCR_DICT_KNOWN_HASH_r31] = _STORE_SUBSCR_DICT_KNOWN_HASH, @@ -5536,6 +5550,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_TOS_LIST_r11] = "_GUARD_TOS_LIST_r11", [_GUARD_TOS_LIST_r22] = "_GUARD_TOS_LIST_r22", [_GUARD_TOS_LIST_r33] = "_GUARD_TOS_LIST_r33", + [_GUARD_TOS_NON_NEGATIVE_COMPACT_INT] = "_GUARD_TOS_NON_NEGATIVE_COMPACT_INT", + [_GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r01] = "_GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r01", + [_GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r11] = "_GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r11", + [_GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r22] = "_GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r22", + [_GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r33] = "_GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r33", [_GUARD_TOS_NOT_NULL] = "_GUARD_TOS_NOT_NULL", [_GUARD_TOS_NOT_NULL_r01] = "_GUARD_TOS_NOT_NULL_r01", [_GUARD_TOS_NOT_NULL_r11] = "_GUARD_TOS_NOT_NULL_r11", @@ -6389,6 +6408,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _STORE_SUBSCR: return 3; + case _GUARD_TOS_NON_NEGATIVE_COMPACT_INT: + return 0; case _STORE_SUBSCR_LIST_INT: return 3; case _STORE_SUBSCR_DICT: diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 7946550ec0db637..f144f0aa348b634 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1890,6 +1890,17 @@ def binary_subscr_tuple_int(): "BINARY_OP_SUBSCR_TUPLE_INT") self.assert_no_opcode(binary_subscr_tuple_int, "BINARY_OP") + def binary_subscr_tuple_negative_int(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = (1, 2, 3) + idx = -1 + self.assertEqual(a[idx], 3) + + binary_subscr_tuple_negative_int() + self.assert_specialized(binary_subscr_tuple_negative_int, + "BINARY_OP_SUBSCR_TUPLE_INT") + self.assert_no_opcode(binary_subscr_tuple_negative_int, "BINARY_OP") + def binary_subscr_dict(): for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a = {1: 2, 2: 3} @@ -1967,6 +1978,16 @@ def binary_subscr_str_int(): self.assert_specialized(binary_subscr_str_int, "BINARY_OP_SUBSCR_STR_INT") self.assert_no_opcode(binary_subscr_str_int, "BINARY_OP") + def binary_subscr_str_negative_int(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = "foobar" + idx = -1 + self.assertEqual(a[idx], "r") + + binary_subscr_str_negative_int() + self.assert_specialized(binary_subscr_str_negative_int, "BINARY_OP_SUBSCR_STR_INT") + self.assert_no_opcode(binary_subscr_str_negative_int, "BINARY_OP") + def binary_subscr_str_int_non_compact(): for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a = "바이트코드_특수화" @@ -1977,6 +1998,16 @@ def binary_subscr_str_int_non_compact(): self.assert_specialized(binary_subscr_str_int_non_compact, "BINARY_OP_SUBSCR_USTR_INT") self.assert_no_opcode(binary_subscr_str_int_non_compact, "BINARY_OP_SUBSCR_STR_INT") + def binary_subscr_str_negative_int_non_compact(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = "바이트코드_특수화" + idx = -1 + self.assertEqual(a[idx], "화") + + binary_subscr_str_negative_int_non_compact() + self.assert_specialized(binary_subscr_str_negative_int_non_compact, "BINARY_OP_SUBSCR_USTR_INT") + self.assert_no_opcode(binary_subscr_str_negative_int_non_compact, "BINARY_OP_SUBSCR_STR_INT") + def binary_subscr_getitems(): class C: def __init__(self, val): @@ -2026,6 +2057,17 @@ def store_subscr_defaultdict(): self.assert_specialized(store_subscr_defaultdict, "STORE_SUBSCR_DICT") self.assert_no_opcode(store_subscr_defaultdict, "STORE_SUBSCR") + def store_subscr_list_negative_int(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = [1, 2, 3] + idx = -1 + a[idx] = 4 + self.assertEqual(a, [1, 2, 4]) + + store_subscr_list_negative_int() + self.assert_specialized(store_subscr_list_negative_int, "STORE_SUBSCR_LIST_INT") + self.assert_no_opcode(store_subscr_list_negative_int, "STORE_SUBSCR") + def store_subscr_dict_subclass_override(): class MyDict(dict): def __setitem__(self, key, value): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-10-10-05-00.gh-issue-153486.compact-index-guard.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-10-10-05-00.gh-issue-153486.compact-index-guard.rst new file mode 100644 index 000000000000000..524461453885e98 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-10-10-05-00.gh-issue-153486.compact-index-guard.rst @@ -0,0 +1,2 @@ +Improve optimized handling of non-negative integer indices for lists, +tuples, and strings. diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index a17648a33d4fe4a..c4ce9416ec2eaf0 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -1036,13 +1036,12 @@ PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); assert(PyLong_CheckExact(sub)); assert(PyUnicode_CheckExact(str)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + Py_ssize_t len = PyUnicode_GET_LENGTH(str); + if (index < 0) { + index += len; } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) { + if (index < 0 || len <= index) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -1119,13 +1118,12 @@ PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); assert(PyLong_CheckExact(sub)); assert(PyTuple_CheckExact(tuple)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + Py_ssize_t len = PyTuple_GET_SIZE(tuple); + if (index < 0) { + index += len; } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) { + if (index < 0 || index >= len) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -1139,7 +1137,10 @@ assert(PyLong_CheckExact(sub)); assert(PyTuple_CheckExact(tuple)); STAT_INC(BINARY_OP, hit); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + if (index < 0) { + index += PyTuple_GET_SIZE(tuple); + } PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); res = PyStackRef_FromPyObjectNew(res_o); @@ -1213,13 +1214,12 @@ PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); assert(PyLong_CheckExact(sub)); assert(PyUnicode_CheckExact(str)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + Py_ssize_t len = PyUnicode_GET_LENGTH(str); + if (index < 0) { + index += len; } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) { + if (index < 0 || len <= index) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4d7b338e2dbd4c3..ae735c61cfbf550 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1134,7 +1134,8 @@ dummy_func( macro(STORE_SLICE) = _SPECIALIZE_STORE_SLICE + _STORE_SLICE; macro(BINARY_OP_SUBSCR_LIST_INT) = - _GUARD_TOS_INT + _GUARD_NOS_LIST + unused/5 + _BINARY_OP_SUBSCR_LIST_INT + _POP_TOP_INT + POP_TOP; + _GUARD_TOS_INT + _GUARD_NOS_LIST + unused/5 + + _BINARY_OP_SUBSCR_LIST_INT + _POP_TOP_INT + POP_TOP; op(_BINARY_OP_SUBSCR_LIST_INT, (list_st, sub_st -- res, ls, ss)) { PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); @@ -1185,7 +1186,8 @@ dummy_func( } macro(BINARY_OP_SUBSCR_STR_INT) = - _GUARD_TOS_INT + _GUARD_NOS_COMPACT_ASCII + unused/5 + _BINARY_OP_SUBSCR_STR_INT + _POP_TOP_INT + _POP_TOP_UNICODE; + _GUARD_TOS_INT + _GUARD_NOS_COMPACT_ASCII + + unused/5 + _BINARY_OP_SUBSCR_STR_INT + _POP_TOP_INT + _POP_TOP_UNICODE; op(_BINARY_OP_SUBSCR_STR_INT, (str_st, sub_st -- res, s, i)) { PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); @@ -1193,9 +1195,12 @@ dummy_func( assert(PyLong_CheckExact(sub)); assert(PyUnicode_CheckExact(str)); - EXIT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - EXIT_IF(PyUnicode_GET_LENGTH(str) <= index); + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + Py_ssize_t len = PyUnicode_GET_LENGTH(str); + if (index < 0) { + index += len; + } + EXIT_IF(index < 0 || len <= index); uint8_t c = PyUnicode_1BYTE_DATA(str)[index]; assert(c < 128); STAT_INC(BINARY_OP, hit); @@ -1207,7 +1212,8 @@ dummy_func( } macro(BINARY_OP_SUBSCR_USTR_INT) = - _GUARD_TOS_INT + _GUARD_NOS_UNICODE + unused/5 + _BINARY_OP_SUBSCR_USTR_INT + _POP_TOP_INT + _POP_TOP_UNICODE; + _GUARD_TOS_INT + _GUARD_NOS_UNICODE + + unused/5 + _BINARY_OP_SUBSCR_USTR_INT + _POP_TOP_INT + _POP_TOP_UNICODE; op(_BINARY_OP_SUBSCR_USTR_INT, (str_st, sub_st -- res, s, i)) { PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); @@ -1215,9 +1221,12 @@ dummy_func( assert(PyLong_CheckExact(sub)); assert(PyUnicode_CheckExact(str)); - EXIT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - EXIT_IF(PyUnicode_GET_LENGTH(str) <= index); + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + Py_ssize_t len = PyUnicode_GET_LENGTH(str); + if (index < 0) { + index += len; + } + EXIT_IF(index < 0 || len <= index); // Specialize for reading an ASCII character from any string: Py_UCS4 c = PyUnicode_READ_CHAR(str, index); EXIT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c); @@ -1256,10 +1265,12 @@ dummy_func( assert(PyLong_CheckExact(sub)); assert(PyTuple_CheckExact(tuple)); - // Deopt unless 0 <= sub < PyTuple_Size(list) - EXIT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - EXIT_IF(index >= PyTuple_GET_SIZE(tuple)); + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + Py_ssize_t len = PyTuple_GET_SIZE(tuple); + if (index < 0) { + index += len; + } + EXIT_IF(index < 0 || index >= len); } op(_BINARY_OP_SUBSCR_TUPLE_INT, (tuple_st, sub_st -- res, ts, ss)) { @@ -1270,7 +1281,10 @@ dummy_func( assert(PyTuple_CheckExact(tuple)); STAT_INC(BINARY_OP, hit); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + if (index < 0) { + index += PyTuple_GET_SIZE(tuple); + } PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); res = PyStackRef_FromPyObjectNew(res_o); @@ -1411,8 +1425,15 @@ dummy_func( macro(STORE_SUBSCR) = _SPECIALIZE_STORE_SUBSCR + _STORE_SUBSCR; + op(_GUARD_TOS_NON_NEGATIVE_COMPACT_INT, (value -- value)) { + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyLong_CheckExact(value_o)); + EXIT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)value_o)); + } + macro(STORE_SUBSCR_LIST_INT) = - _GUARD_TOS_INT + _GUARD_NOS_LIST + unused/1 + _STORE_SUBSCR_LIST_INT + _POP_TOP_INT + POP_TOP; + _GUARD_TOS_INT + _GUARD_NOS_LIST + unused/1 + + _STORE_SUBSCR_LIST_INT + _POP_TOP_INT + POP_TOP; op(_STORE_SUBSCR_LIST_INT, (value, list_st, sub_st -- ls, ss)) { PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); @@ -1424,7 +1445,6 @@ dummy_func( Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); DEOPT_IF(!LOCK_OBJECT(list)); Py_ssize_t len = PyList_GET_SIZE(list); - // Ensure index < len(list) if (index < 0) { index += len; } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e45bbd7cceb295f..21427952059cd39 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7018,15 +7018,12 @@ PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); assert(PyLong_CheckExact(sub)); assert(PyUnicode_CheckExact(str)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = sub_st; - _tos_cache0 = str_st; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + Py_ssize_t len = PyUnicode_GET_LENGTH(str); + if (index < 0) { + index += len; } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) { + if (index < 0 || len <= index) { UOP_STAT_INC(uopcode, miss); _tos_cache1 = sub_st; _tos_cache0 = str_st; @@ -7064,15 +7061,12 @@ PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); assert(PyLong_CheckExact(sub)); assert(PyUnicode_CheckExact(str)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = sub_st; - _tos_cache0 = str_st; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + Py_ssize_t len = PyUnicode_GET_LENGTH(str); + if (index < 0) { + index += len; } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) { + if (index < 0 || len <= index) { UOP_STAT_INC(uopcode, miss); _tos_cache1 = sub_st; _tos_cache0 = str_st; @@ -7285,13 +7279,12 @@ PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); assert(PyLong_CheckExact(sub)); assert(PyTuple_CheckExact(tuple)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + Py_ssize_t len = PyTuple_GET_SIZE(tuple); + if (index < 0) { + index += len; } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) { + if (index < 0 || index >= len) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); @@ -7317,14 +7310,12 @@ PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); assert(PyLong_CheckExact(sub)); assert(PyTuple_CheckExact(tuple)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = sub_st; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + Py_ssize_t len = PyTuple_GET_SIZE(tuple); + if (index < 0) { + index += len; } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) { + if (index < 0 || index >= len) { UOP_STAT_INC(uopcode, miss); _tos_cache0 = sub_st; SET_CURRENT_CACHED_VALUES(1); @@ -7352,15 +7343,12 @@ PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); assert(PyLong_CheckExact(sub)); assert(PyTuple_CheckExact(tuple)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache1 = sub_st; - _tos_cache0 = tuple_st; - SET_CURRENT_CACHED_VALUES(2); - JUMP_TO_JUMP_TARGET(); + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + Py_ssize_t len = PyTuple_GET_SIZE(tuple); + if (index < 0) { + index += len; } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) { + if (index < 0 || index >= len) { UOP_STAT_INC(uopcode, miss); _tos_cache1 = sub_st; _tos_cache0 = tuple_st; @@ -7388,16 +7376,12 @@ PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); assert(PyLong_CheckExact(sub)); assert(PyTuple_CheckExact(tuple)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache2 = sub_st; - _tos_cache1 = tuple_st; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - JUMP_TO_JUMP_TARGET(); + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + Py_ssize_t len = PyTuple_GET_SIZE(tuple); + if (index < 0) { + index += len; } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) { + if (index < 0 || index >= len) { UOP_STAT_INC(uopcode, miss); _tos_cache2 = sub_st; _tos_cache1 = tuple_st; @@ -7428,7 +7412,10 @@ assert(PyLong_CheckExact(sub)); assert(PyTuple_CheckExact(tuple)); STAT_INC(BINARY_OP, hit); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + if (index < 0) { + index += PyTuple_GET_SIZE(tuple); + } PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); res = PyStackRef_FromPyObjectNew(res_o); @@ -7460,7 +7447,10 @@ assert(PyLong_CheckExact(sub)); assert(PyTuple_CheckExact(tuple)); STAT_INC(BINARY_OP, hit); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + if (index < 0) { + index += PyTuple_GET_SIZE(tuple); + } PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); res = PyStackRef_FromPyObjectNew(res_o); @@ -7493,7 +7483,10 @@ assert(PyLong_CheckExact(sub)); assert(PyTuple_CheckExact(tuple)); STAT_INC(BINARY_OP, hit); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + if (index < 0) { + index += PyTuple_GET_SIZE(tuple); + } PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); res = PyStackRef_FromPyObjectNew(res_o); @@ -8341,6 +8334,95 @@ break; } + case _GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r01: { + CHECK_CURRENT_CACHED_VALUES(0); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef value; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyLong_CheckExact(value_o)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)value_o)) { + UOP_STAT_INC(uopcode, miss); + SET_CURRENT_CACHED_VALUES(0); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + stack_pointer += -1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r11: { + CHECK_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + value = _stack_item_0; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyLong_CheckExact(value_o)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)value_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache0 = value; + SET_CURRENT_CACHED_VALUES(1); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r22: { + CHECK_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + value = _stack_item_1; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyLong_CheckExact(value_o)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)value_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache1 = value; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(2); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + + case _GUARD_TOS_NON_NEGATIVE_COMPACT_INT_r33: { + CHECK_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + _PyStackRef value; + _PyStackRef _stack_item_0 = _tos_cache0; + _PyStackRef _stack_item_1 = _tos_cache1; + _PyStackRef _stack_item_2 = _tos_cache2; + value = _stack_item_2; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyLong_CheckExact(value_o)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)value_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + JUMP_TO_JUMP_TARGET(); + } + _tos_cache2 = value; + _tos_cache1 = _stack_item_1; + _tos_cache0 = _stack_item_0; + SET_CURRENT_CACHED_VALUES(3); + ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); + break; + } + case _STORE_SUBSCR_LIST_INT_r32: { CHECK_CURRENT_CACHED_VALUES(3); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6178dc70c1b80e7..69822daf8c512c9 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1036,13 +1036,12 @@ PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); assert(PyLong_CheckExact(sub)); assert(PyUnicode_CheckExact(str)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + Py_ssize_t len = PyUnicode_GET_LENGTH(str); + if (index < 0) { + index += len; } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) { + if (index < 0 || len <= index) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -1119,13 +1118,12 @@ PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); assert(PyLong_CheckExact(sub)); assert(PyTuple_CheckExact(tuple)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + Py_ssize_t len = PyTuple_GET_SIZE(tuple); + if (index < 0) { + index += len; } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) { + if (index < 0 || index >= len) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -1139,7 +1137,10 @@ assert(PyLong_CheckExact(sub)); assert(PyTuple_CheckExact(tuple)); STAT_INC(BINARY_OP, hit); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + if (index < 0) { + index += PyTuple_GET_SIZE(tuple); + } PyObject *res_o = PyTuple_GET_ITEM(tuple, index); assert(res_o != NULL); res = PyStackRef_FromPyObjectNew(res_o); @@ -1213,13 +1214,12 @@ PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); assert(PyLong_CheckExact(sub)); assert(PyUnicode_CheckExact(str)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject*)sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + Py_ssize_t len = PyUnicode_GET_LENGTH(str); + if (index < 0) { + index += len; } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) { + if (index < 0 || len <= index) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index edb4c644bccbf6f..5a558a70349c9a7 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -598,9 +598,11 @@ dummy_func(void) { if (sym_is_const(ctx, sub_st)) { assert(PyLong_CheckExact(sym_get_const(ctx, sub_st))); long index = PyLong_AsLong(sym_get_const(ctx, sub_st)); - assert(index >= 0); Py_ssize_t tuple_length = sym_tuple_length(tuple_st); - if (tuple_length != -1 && index < tuple_length) { + if (tuple_length != -1 && index < 0) { + index += tuple_length; + } + if (tuple_length != -1 && 0 <= index && index < tuple_length) { ADD_OP(_NOP, 0, 0); } } @@ -611,14 +613,17 @@ dummy_func(void) { if (sym_is_const(ctx, sub_st)) { assert(PyLong_CheckExact(sym_get_const(ctx, sub_st))); long index = PyLong_AsLong(sym_get_const(ctx, sub_st)); - assert(index >= 0); Py_ssize_t tuple_length = sym_tuple_length(tuple_st); if (tuple_length == -1) { // Unknown length res = sym_new_not_null(ctx); } else { + if (index < 0) { + index += tuple_length; + } assert(index < tuple_length); + assert(index >= 0); res = sym_tuple_getitem(ctx, tuple_st, index); } } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 5e110360b81b445..13b4dcb1e23cbd1 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1430,9 +1430,11 @@ if (sym_is_const(ctx, sub_st)) { assert(PyLong_CheckExact(sym_get_const(ctx, sub_st))); long index = PyLong_AsLong(sym_get_const(ctx, sub_st)); - assert(index >= 0); Py_ssize_t tuple_length = sym_tuple_length(tuple_st); - if (tuple_length != -1 && index < tuple_length) { + if (tuple_length != -1 && index < 0) { + index += tuple_length; + } + if (tuple_length != -1 && 0 <= index && index < tuple_length) { ADD_OP(_NOP, 0, 0); } } @@ -1451,13 +1453,16 @@ if (sym_is_const(ctx, sub_st)) { assert(PyLong_CheckExact(sym_get_const(ctx, sub_st))); long index = PyLong_AsLong(sym_get_const(ctx, sub_st)); - assert(index >= 0); Py_ssize_t tuple_length = sym_tuple_length(tuple_st); if (tuple_length == -1) { res = sym_new_not_null(ctx); } else { + if (index < 0) { + index += tuple_length; + } assert(index < tuple_length); + assert(index >= 0); res = sym_tuple_getitem(ctx, tuple_st, index); } } @@ -1719,6 +1724,10 @@ break; } + case _GUARD_TOS_NON_NEGATIVE_COMPACT_INT: { + break; + } + case _STORE_SUBSCR_LIST_INT: { JitOptRef sub_st; JitOptRef list_st; diff --git a/Python/specialize.c b/Python/specialize.c index 773b55c329c7720..20e9cdbc2442843 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1518,7 +1518,7 @@ store_subscr_fail_kind(PyObject *container, PyObject *sub) return SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE; } if (PyObject_CheckBuffer(container)) { - if (PyLong_CheckExact(sub) && (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub))) { + if (PyLong_CheckExact(sub) && !_PyLong_CheckExactAndCompact(sub)) { return SPEC_FAIL_OUT_OF_RANGE; } else if (strcmp(container_type->tp_name, "array.array") == 0) { @@ -1582,17 +1582,20 @@ _Py_Specialize_StoreSubscr(_PyStackRef container_st, _PyStackRef sub_st, _Py_COD PyTypeObject *container_type = Py_TYPE(container); if (container_type == &PyList_Type) { if (PyLong_CheckExact(sub)) { - if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub) - && ((PyLongObject *)sub)->long_value.ob_digit[0] < (size_t)PyList_GET_SIZE(container)) - { - specialize(instr, STORE_SUBSCR_LIST_INT); - return; - } - else { - SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); - unspecialize(instr); - return; + if (_PyLong_CheckExactAndCompact(sub)) { + Py_ssize_t index = _PyLong_CompactValue((PyLongObject *)sub); + Py_ssize_t len = PyList_GET_SIZE(container); + if (index < 0) { + index += len; + } + if (0 <= index && index < len) { + specialize(instr, STORE_SUBSCR_LIST_INT); + return; + } } + SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); + unspecialize(instr); + return; } else if (PySlice_Check(sub)) { SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_LIST_SLICE); @@ -2015,7 +2018,9 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) return SPEC_FAIL_BINARY_OP_XOR; case NB_SUBSCR: if (PyList_CheckExact(lhs)) { - if (PyLong_CheckExact(rhs) && !_PyLong_IsNonNegativeCompact((PyLongObject *)rhs)) { + if (PyLong_CheckExact(rhs) && + !_PyLong_CheckExactAndCompact(rhs)) + { return SPEC_FAIL_OUT_OF_RANGE; } if (PySlice_Check(rhs)) { @@ -2023,7 +2028,7 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) } } if (PyTuple_CheckExact(lhs)) { - if (PyLong_CheckExact(rhs) && !_PyLong_IsNonNegativeCompact((PyLongObject *)rhs)) { + if (PyLong_CheckExact(rhs) && !_PyLong_CheckExactAndCompact(rhs)) { return SPEC_FAIL_OUT_OF_RANGE; } if (PySlice_Check(rhs)) { @@ -2031,7 +2036,7 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs) } } if (PyUnicode_CheckExact(lhs)) { - if (PyLong_CheckExact(rhs) && !_PyLong_IsNonNegativeCompact((PyLongObject *)rhs)) { + if (PyLong_CheckExact(rhs) && !_PyLong_CheckExactAndCompact(rhs)) { return SPEC_FAIL_OUT_OF_RANGE; } if (PySlice_Check(rhs)) { @@ -2398,16 +2403,16 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in } break; case NB_SUBSCR: - if (PyLong_CheckExact(rhs) && _PyLong_IsNonNegativeCompact((PyLongObject *)rhs)) { - if (PyList_CheckExact(lhs)) { - specialize(instr, BINARY_OP_SUBSCR_LIST_INT); - return; - } + if (_PyLong_CheckExactAndCompact(rhs) && PyList_CheckExact(lhs)) { + specialize(instr, BINARY_OP_SUBSCR_LIST_INT); + return; + } + if (_PyLong_CheckExactAndCompact(rhs)) { if (PyTuple_CheckExact(lhs)) { specialize(instr, BINARY_OP_SUBSCR_TUPLE_INT); return; } - if (PyUnicode_CheckExact(lhs) && _PyLong_IsNonNegativeCompact((PyLongObject*)rhs)) { + if (PyUnicode_CheckExact(lhs)) { if (PyUnicode_IS_COMPACT_ASCII(lhs)) { specialize(instr, BINARY_OP_SUBSCR_STR_INT); return;