diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index fb5622c99f7a135..157fe871e71a0ad 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -116,6 +116,7 @@ PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, int64_t); PyAPI_FUNC(_PyStackRef) _PyCompactLong_Add(PyLongObject *left, PyLongObject *right); PyAPI_FUNC(_PyStackRef) _PyCompactLong_Multiply(PyLongObject *left, PyLongObject *right); PyAPI_FUNC(_PyStackRef) _PyCompactLong_Subtract(PyLongObject *left, PyLongObject *right); +PyAPI_FUNC(_PyStackRef) _PyLong_SubtractShrink(PyLongObject *left, PyLongObject *right); // Export for 'binascii' shared extension. PyAPI_DATA(unsigned char) _PyLong_DigitValue[256]; @@ -346,6 +347,28 @@ _PyLong_CheckExactAndCompact(PyObject *op) return PyLong_CheckExact(op) && _PyLong_IsCompact((const PyLongObject *)op); } +static inline bool +_PyLong_IsShrinkSubtractPair(const PyLongObject *left, const PyLongObject *right) +{ +#if PyLong_SHIFT == 30 + if (_PyLong_IsCompact((PyLongObject *)left) || + _PyLong_IsCompact((PyLongObject *)right) || + !_PyLong_SameSign(left, right)) { + return false; + } + Py_ssize_t left_size = _PyLong_DigitCount(left); + if (left_size != _PyLong_DigitCount(right) || + left_size < 2 || + left_size > 3) { + return false; + } + return left->long_value.ob_digit[left_size - 1] == + right->long_value.ob_digit[left_size - 1]; +#else + return false; +#endif +} + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 31d2e31573d8c53..ed2bf1781890e66 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1116,7 +1116,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BINARY_OP_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_USTR_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, [BINARY_OP_SUBTRACT_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, - [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, + [BINARY_OP_SUBTRACT_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_INTERPOLATION] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1371,7 +1371,7 @@ _PyOpcode_macro_expansion[256] = { [BINARY_OP_SUBSCR_TUPLE_INT] = { .nuops = 6, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_TUPLE, OPARG_SIMPLE, 0 }, { _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_TUPLE_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, [BINARY_OP_SUBSCR_USTR_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_USTR_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 5 } } }, [BINARY_OP_SUBTRACT_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBTRACT_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBTRACT_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBTRACT_INT] = { .nuops = 3, .uops = { { _BINARY_OP_SUBTRACT_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, [BINARY_SLICE] = { .nuops = 1, .uops = { { _BINARY_SLICE, OPARG_SIMPLE, 0 } } }, [BUILD_INTERPOLATION] = { .nuops = 1, .uops = { { _BUILD_INTERPOLATION, OPARG_SIMPLE, 0 } } }, [BUILD_LIST] = { .nuops = 1, .uops = { { _BUILD_LIST, OPARG_SIMPLE, 0 } } }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 97f934727a85a36..13bf37595122ca7 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -506,8 +506,6 @@ enum { _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13, _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23, - _BINARY_OP_SUBTRACT_INT_r03, - _BINARY_OP_SUBTRACT_INT_r13, _BINARY_OP_SUBTRACT_INT_r23, _BINARY_OP_SUBTRACT_INT_INPLACE_r03, _BINARY_OP_SUBTRACT_INT_INPLACE_r13, @@ -1434,7 +1432,7 @@ enum { _WITH_EXCEPT_START_r33, _YIELD_VALUE_r11, }; -#define MAX_UOP_REGS_ID 1645 +#define MAX_UOP_REGS_ID 1643 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 990706c0b329223..4eb12273560d4b3 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -107,7 +107,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_TOS_OVERFLOWED] = HAS_EXIT_FLAG, [_BINARY_OP_MULTIPLY_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, - [_BINARY_OP_SUBTRACT_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_SUBTRACT_INT] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_BINARY_OP_ADD_INT_INPLACE] = HAS_EXIT_FLAG, [_BINARY_OP_SUBTRACT_INT_INPLACE] = HAS_EXIT_FLAG, [_BINARY_OP_MULTIPLY_INT_INPLACE] = HAS_EXIT_FLAG, @@ -1093,10 +1093,10 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { }, }, [_BINARY_OP_SUBTRACT_INT] = { - .best = { 0, 1, 2, 2 }, + .best = { 2, 2, 2, 2 }, .entries = { - { 3, 0, _BINARY_OP_SUBTRACT_INT_r03 }, - { 3, 1, _BINARY_OP_SUBTRACT_INT_r13 }, + { -1, -1, -1 }, + { -1, -1, -1 }, { 3, 2, _BINARY_OP_SUBTRACT_INT_r23 }, { -1, -1, -1 }, }, @@ -4109,8 +4109,6 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_BINARY_OP_ADD_INT_r03] = _BINARY_OP_ADD_INT, [_BINARY_OP_ADD_INT_r13] = _BINARY_OP_ADD_INT, [_BINARY_OP_ADD_INT_r23] = _BINARY_OP_ADD_INT, - [_BINARY_OP_SUBTRACT_INT_r03] = _BINARY_OP_SUBTRACT_INT, - [_BINARY_OP_SUBTRACT_INT_r13] = _BINARY_OP_SUBTRACT_INT, [_BINARY_OP_SUBTRACT_INT_r23] = _BINARY_OP_SUBTRACT_INT, [_BINARY_OP_ADD_INT_INPLACE_r03] = _BINARY_OP_ADD_INT_INPLACE, [_BINARY_OP_ADD_INT_INPLACE_r13] = _BINARY_OP_ADD_INT_INPLACE, @@ -4922,8 +4920,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13", [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23] = "_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23", [_BINARY_OP_SUBTRACT_INT] = "_BINARY_OP_SUBTRACT_INT", - [_BINARY_OP_SUBTRACT_INT_r03] = "_BINARY_OP_SUBTRACT_INT_r03", - [_BINARY_OP_SUBTRACT_INT_r13] = "_BINARY_OP_SUBTRACT_INT_r13", [_BINARY_OP_SUBTRACT_INT_r23] = "_BINARY_OP_SUBTRACT_INT_r23", [_BINARY_OP_SUBTRACT_INT_INPLACE] = "_BINARY_OP_SUBTRACT_INT_INPLACE", [_BINARY_OP_SUBTRACT_INT_INPLACE_r03] = "_BINARY_OP_SUBTRACT_INT_INPLACE_r03", diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 25b2c393e6773de..c5d5d3f013ec298 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -4455,6 +4455,22 @@ def testfunc(n): self.assertIn("_POP_TOP_NOP", uops) self.assertLessEqual(count_ops(ex, "_POP_TOP"), 2) + def test_int_subtract_shrink_specialization(self): + def testfunc(args): + a, b, n = args + res = 0 + for _ in range(n): + res = a - b + return res + + res, ex = self._run_with_optimizer( + testfunc, (10_000_000_001, 10_000_000_000, TIER2_THRESHOLD) + ) + self.assertEqual(res, 1) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + self.assertIn("_BINARY_OP_SUBTRACT_INT", uops) + def test_int_mul_op_refcount_elimination(self): def testfunc(n): c = 1 diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index b48a8812a1a2d13..1628d1038c41246 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -1140,6 +1140,25 @@ def test_small_ints(self): self.assertIs(i - i, 0) self.assertIs(0 * i, 0) + @support.cpython_only + def test_int64_boundary_sub(self): + INT64_MAX = (1 << 63) - 1 + INT64_MIN = -(1 << 63) + + self.assertEqual(INT64_MIN - 0, INT64_MIN) + self.assertEqual(INT64_MIN - (-1), INT64_MIN + 1) + + self.assertEqual(INT64_MAX - 0, INT64_MAX) + self.assertEqual(INT64_MAX - 1, INT64_MAX - 1) + + self.assertEqual(INT64_MIN - 1, INT64_MIN - 1) + self.assertEqual(INT64_MAX - (-1), 1 << 63) + + beyond_max = INT64_MAX + 2 + beyond_min = INT64_MIN - 2 + self.assertEqual(beyond_max - 1, INT64_MAX + 1) + self.assertEqual(beyond_min - (-1), INT64_MIN - 1) + def test_bit_length(self): tiny = 1e-10 for x in range(-65000, 65000): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-10-17-40-41.gh-issue-151289.xZhM7W.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-10-17-40-41.gh-issue-151289.xZhM7W.rst new file mode 100644 index 000000000000000..8285ad1e0b3064f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-10-17-40-41.gh-issue-151289.xZhM7W.rst @@ -0,0 +1,2 @@ +Improve performance of binary ``int`` subtraction for shrinking wide exact +integers. diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index a17648a33d4fe4a..949c3d182f9e439 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -1340,43 +1340,52 @@ next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; _PyStackRef left; _PyStackRef right; _PyStackRef res; _PyStackRef l; _PyStackRef r; - // _GUARD_TOS_INT + _PyStackRef value; + /* Skip 5 cache entries */ + // _BINARY_OP_SUBTRACT_INT { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyLong_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - } - // _GUARD_NOS_INT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + bool both_compact = _PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o); + bool shrink_pair = false; + if (!both_compact) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + shrink_pair = _PyLong_IsShrinkSubtractPair((PyLongObject *)left_o, (PyLongObject *)right_o); + _PyFrame_StackPointerInvalidate(frame); + } + if (!(both_compact || shrink_pair)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBTRACT_INT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (both_compact) { + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + res = _PyLong_SubtractShrink((PyLongObject *)left_o, (PyLongObject *)right_o); + _PyFrame_StackPointerInvalidate(frame); + } if (PyStackRef_IsNull(res)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); diff --git a/Objects/longobject.c b/Objects/longobject.c index 7a38ae8ea5a36f0..e85c6e2f0c1aac0 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3916,6 +3916,72 @@ _PyCompactLong_Subtract(PyLongObject *a, PyLongObject *b) return medium_from_stwodigits(v); } +static inline _PyStackRef +shrink_subtract_result(int sign, const digit *digits, Py_ssize_t size) +{ + while (size > 0 && digits[size - 1] == 0) { + --size; + } + if (size == 0) { + return PyStackRef_FromPyObjectBorrow(_PyLong_GetZero()); + } + if (size == 1) { + stwodigits value = (stwodigits)digits[0]; + if (sign < 0) { + value = -value; + } + return medium_from_stwodigits(value); + } + PyLongObject *result = PyObject_Malloc( + offsetof(PyLongObject, long_value.ob_digit) + size * sizeof(digit)); + if (result == NULL) { + return PyStackRef_NULL; + } + _PyObject_Init((PyObject *)result, &PyLong_Type); + _PyLong_InitTag(result); + _PyLong_SetSignAndDigitCount(result, sign, size); + memcpy(result->long_value.ob_digit, digits, size * sizeof(digit)); + return PyStackRef_FromPyObjectStealMortal((PyObject *)result); +} + +_PyStackRef +_PyLong_SubtractShrink(PyLongObject *left, PyLongObject *right) +{ + if (!_PyLong_IsShrinkSubtractPair(left, right)) { + return PyStackRef_NULL; + } + Py_ssize_t size = _PyLong_DigitCount(left); + assert(size >= 2 && size <= 3); + + int sign = _PyLong_IsNegative(left) ? -1 : 1; + int cmp = 0; + for (Py_ssize_t i = size - 2; i >= 0; i--) { + digit left_digit = left->long_value.ob_digit[i]; + digit right_digit = right->long_value.ob_digit[i]; + if (left_digit != right_digit) { + cmp = left_digit > right_digit ? 1 : -1; + break; + } + } + if (cmp == 0) { + return PyStackRef_FromPyObjectBorrow(_PyLong_GetZero()); + } + + PyLongObject *larger = cmp > 0 ? left : right; + PyLongObject *smaller = cmp > 0 ? right : left; + digit out[3]; + twodigits borrow = 0; + for (Py_ssize_t i = 0; i < size; i++) { + twodigits lhs = (twodigits)larger->long_value.ob_digit[i]; + twodigits rhs = (twodigits)smaller->long_value.ob_digit[i]; + twodigits diff = lhs - rhs - borrow; + out[i] = (digit)(diff & PyLong_MASK); + borrow = lhs < rhs + borrow; + } + assert(borrow == 0); + return shrink_subtract_result(cmp > 0 ? sign : -sign, out, size); +} + static PyObject * long_sub_method(PyObject *a, PyObject *b) { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 4d7b338e2dbd4c3..e4daf98a194b155 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -688,12 +688,22 @@ dummy_func( pure op(_BINARY_OP_SUBTRACT_INT, (left, right -- res, l, r)) { PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + EXIT_IF(!PyLong_CheckExact(left_o)); + EXIT_IF(!PyLong_CheckExact(right_o)); + bool both_compact = _PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o); + bool shrink_pair = false; + if (!both_compact) { + shrink_pair = _PyLong_IsShrinkSubtractPair((PyLongObject *)left_o, (PyLongObject *)right_o); + } + EXIT_IF(!(both_compact || shrink_pair)); STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (both_compact) { + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + } + else { + res = _PyLong_SubtractShrink((PyLongObject *)left_o, (PyLongObject *)right_o); + } EXIT_IF(PyStackRef_IsNull(res)); l = left; r = right; @@ -707,7 +717,7 @@ dummy_func( _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_ADD_INT + _POP_TOP_INT + _POP_TOP_INT; macro(BINARY_OP_SUBTRACT_INT) = - _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_SUBTRACT_INT + _POP_TOP_INT + _POP_TOP_INT; + unused/5 + _BINARY_OP_SUBTRACT_INT + _POP_TOP_INT + _POP_TOP_INT; // Inplace compact int ops: mutate the uniquely-referenced operand // when possible. The op handles decref of TARGET internally so diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e45bbd7cceb295f..59ce53c911436a5 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4403,76 +4403,6 @@ break; } - case _BINARY_OP_SUBTRACT_INT_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UOP_STAT_INC(uopcode, miss); - SET_CURRENT_CACHED_VALUES(0); - JUMP_TO_JUMP_TARGET(); - } - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - break; - } - - case _BINARY_OP_SUBTRACT_INT_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - _PyStackRef right; - _PyStackRef left; - _PyStackRef res; - _PyStackRef l; - _PyStackRef r; - _PyStackRef _stack_item_0 = _tos_cache0; - right = _stack_item_0; - left = stack_pointer[-1]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UOP_STAT_INC(uopcode, miss); - _tos_cache0 = right; - SET_CURRENT_CACHED_VALUES(1); - JUMP_TO_JUMP_TARGET(); - } - l = left; - r = right; - _tos_cache2 = r; - _tos_cache1 = l; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); - break; - } - case _BINARY_OP_SUBTRACT_INT_r23: { CHECK_CURRENT_CACHED_VALUES(2); ASSERT_WITHIN_STACK_BOUNDS_IGNORING_CACHE(__FILE__, __LINE__); @@ -4487,11 +4417,55 @@ left = _stack_item_0; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + if (!PyLong_CheckExact(left_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + if (!PyLong_CheckExact(right_o)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } + bool both_compact = _PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o); + bool shrink_pair = false; + if (!both_compact) { + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + shrink_pair = _PyLong_IsShrinkSubtractPair((PyLongObject *)left_o, (PyLongObject *)right_o); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + } + if (!(both_compact || shrink_pair)) { + UOP_STAT_INC(uopcode, miss); + _tos_cache1 = right; + _tos_cache0 = left; + SET_CURRENT_CACHED_VALUES(2); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (both_compact) { + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + } + else { + stack_pointer[0] = left; + stack_pointer[1] = right; + stack_pointer += 2; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + res = _PyLong_SubtractShrink((PyLongObject *)left_o, (PyLongObject *)right_o); + _PyFrame_StackPointerInvalidate(frame); + stack_pointer += -2; + } if (PyStackRef_IsNull(res)) { UOP_STAT_INC(uopcode, miss); _tos_cache1 = right; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6178dc70c1b80e7..b6da5bba405f761 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1340,43 +1340,52 @@ next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; _PyStackRef left; _PyStackRef right; _PyStackRef res; _PyStackRef l; _PyStackRef r; - // _GUARD_TOS_INT + _PyStackRef value; + /* Skip 5 cache entries */ + // _BINARY_OP_SUBTRACT_INT { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + if (!PyLong_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - } - // _GUARD_NOS_INT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { + if (!PyLong_CheckExact(right_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + bool both_compact = _PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o); + bool shrink_pair = false; + if (!both_compact) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + shrink_pair = _PyLong_IsShrinkSubtractPair((PyLongObject *)left_o, (PyLongObject *)right_o); + _PyFrame_StackPointerInvalidate(frame); + } + if (!(both_compact || shrink_pair)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBTRACT_INT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (both_compact) { + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + res = _PyLong_SubtractShrink((PyLongObject *)left_o, (PyLongObject *)right_o); + _PyFrame_StackPointerInvalidate(frame); + } if (PyStackRef_IsNull(res)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 5e110360b81b445..e785a52e3c5d5bb 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -762,11 +762,30 @@ /* Start of uop copied from bytecodes for constant evaluation */ PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + if (!PyLong_CheckExact(left_o)) { + ctx->done = true; + break; + } + if (!PyLong_CheckExact(right_o)) { + ctx->done = true; + break; + } + bool both_compact = _PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o); + bool shrink_pair = false; + if (!both_compact) { + shrink_pair = _PyLong_IsShrinkSubtractPair((PyLongObject *)left_o, (PyLongObject *)right_o); + } + if (!(both_compact || shrink_pair)) { + ctx->done = true; + break; + } STAT_INC(BINARY_OP, hit); - res_stackref = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (both_compact) { + res_stackref = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + } + else { + res_stackref = _PyLong_SubtractShrink((PyLongObject *)left_o, (PyLongObject *)right_o); + } if (PyStackRef_IsNull(res_stackref )) { ctx->done = true; break; diff --git a/Python/specialize.c b/Python/specialize.c index 773b55c329c7720..f3d20fea75c6730 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2388,7 +2388,9 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) { break; } - if (_PyLong_CheckExactAndCompact(lhs) && _PyLong_CheckExactAndCompact(rhs)) { + if (PyLong_CheckExact(lhs) && PyLong_CheckExact(rhs) && + (_PyLong_BothAreCompact((PyLongObject *)lhs, (PyLongObject *)rhs) || + _PyLong_IsShrinkSubtractPair((PyLongObject *)lhs, (PyLongObject *)rhs))) { specialize(instr, BINARY_OP_SUBTRACT_INT); return; }