From bf8433c825c84eb6f2e7b0cb5d7c462e0e5cd1b8 Mon Sep 17 00:00:00 2001 From: ndossche <7771979+ndossche@users.noreply.github.com> Date: Mon, 24 Aug 2026 21:31:46 +0200 Subject: [PATCH] Speed up call frame leaving by grouping refcounted CVs Leaving the call frame must destroy all the CVs. It loops over all the vars and then uses i_zval_ptr_dtor on them. We can skip the non-refcounted CVs in bulk to speed up that loop. --- Zend/Optimizer/compact_vars.c | 111 ++++++++++++++++++++--- Zend/Optimizer/zend_dump.c | 3 + Zend/Optimizer/zend_optimizer.c | 8 +- Zend/Optimizer/zend_optimizer_internal.h | 2 +- Zend/zend_compile.h | 2 + Zend/zend_execute.c | 2 +- Zend/zend_object_handlers.c | 1 + Zend/zend_opcode.c | 4 + ext/opcache/jit/zend_jit.c | 6 +- ext/opcache/jit/zend_jit_trace.c | 5 +- ext/opcache/tests/opt/coalesce.phpt | 4 +- ext/opcache/tests/opt/sccp_022.phpt | 2 +- ext/opcache/tests/opt/sccp_024.phpt | 2 +- ext/opcache/tests/opt/sccp_026.phpt | 2 +- 14 files changed, 126 insertions(+), 28 deletions(-) diff --git a/Zend/Optimizer/compact_vars.c b/Zend/Optimizer/compact_vars.c index f6c39e9fe166..429c2abd5a51 100644 --- a/Zend/Optimizer/compact_vars.c +++ b/Zend/Optimizer/compact_vars.c @@ -18,17 +18,69 @@ #include "zend_bitset.h" #include "zend_observer.h" +/* Types a CV may have, excluding array kinds. */ +#define CV_TYPE_KINDS (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) +/* Type kinds that are not refcounted. */ +#define CV_TYPE_KINDS_NO_RC (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_BOOL|MAY_BE_LONG|MAY_BE_DOUBLE) + +/* Fills in `rc_cvs` with the CV numbers that may be refcounted, based on SSA info (if available). */ +static void compute_rc_cvs( + const zend_op_array *op_array, const zend_ssa *ssa, zend_bitset rc_cvs) +{ + if (!ssa || !ssa->var_info) { + /* No type information: every CV has to be assumed refcounted. */ + zend_bitset_fill(rc_cvs, zend_bitset_len(op_array->last_var)); + return; + } + + zend_bitset_clear(rc_cvs, zend_bitset_len(op_array->last_var)); + + /* TODO: Argument slots right now have to be always destroyed because + * when ZEND_RECV fails, they can be refcounted (e.g. passing an array to an int parameter). + * This can be improved by making the ZEND_RECV op destroy the argument iself. */ + for (uint32_t i = 0; i < MIN((uint32_t) op_array->last_var, op_array->num_args); i++) { + zend_bitset_incl(rc_cvs, i); + } + for (int i = 0; i < ssa->vars_count; i++) { + int cv = ssa->vars[i].var; + + if (cv < 0 || cv >= op_array->last_var || zend_bitset_in(rc_cvs, cv)) { + continue; + } + if (ssa->var_info[i].type & CV_TYPE_KINDS & ~CV_TYPE_KINDS_NO_RC) { + zend_bitset_incl(rc_cvs, cv); + } + } + + /* ZEND_RETURN_BY_REF turns its op1 into a reference without being recorded as + * such in the SSA. */ + /* TODO: should this be recorded by type inference and SSA construction via a new def? */ + if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) { + for (uint32_t i = 0; i < op_array->last; i++) { + const zend_op *opline = &op_array->opcodes[i]; + + if (opline->opcode == ZEND_RETURN_BY_REF && opline->op1_type == IS_CV) { + zend_bitset_incl(rc_cvs, EX_VAR_TO_NUM(opline->op1.var)); + } + } + } +} + /* This pass removes all CVs and temporaries that are completely unused. It does *not* merge any CVs or TMPs. - * This pass does not operate on SSA form anymore. */ -void zend_optimizer_compact_vars(zend_op_array *op_array) { + * + * It also sorts the refcounted CVs so they appear at the front, to fill in `op_array->last_var_to_free`. + * This improves call frame cleanup performance by skipping (in bulk) the CVs that are not refcounted. */ +void zend_optimizer_compact_vars(zend_op_array *op_array, const zend_ssa *ssa) { int i; ALLOCA_FLAG(use_heap1); ALLOCA_FLAG(use_heap2); + ALLOCA_FLAG(use_heap3); uint32_t used_vars_len = zend_bitset_len(op_array->last_var + op_array->T); zend_bitset used_vars = ZEND_BITSET_ALLOCA(used_vars_len, use_heap1); uint32_t *vars_map = do_alloca((op_array->last_var + op_array->T) * sizeof(uint32_t), use_heap2); - uint32_t num_cvs, num_tmps; + uint32_t rc_cvs_len = zend_bitset_len(op_array->last_var); + zend_bitset rc_cvs = ZEND_BITSET_ALLOCA(rc_cvs_len, use_heap3); /* Determine which CVs are used */ zend_bitset_clear(used_vars, used_vars_len); @@ -52,16 +104,36 @@ void zend_optimizer_compact_vars(zend_op_array *op_array) { } } - num_cvs = 0; + compute_rc_cvs(op_array, ssa, rc_cvs); + + uint32_t num_cvs = 0; + uint32_t last_var_to_free = 0; for (i = 0; i < op_array->last_var; i++) { - if (zend_bitset_in(used_vars, i)) { - vars_map[i] = num_cvs++; - } else { + /* Parameters have a fixed position because other components depend on that ordering. */ + if (i < op_array->num_args) { + ZEND_ASSERT(zend_bitset_in(used_vars, i) + && "A parameter CV is written by its RECV and cannot be unused"); + } else if (!zend_bitset_in(used_vars, i) || !zend_bitset_in(rc_cvs, i)) { vars_map[i] = (uint32_t) -1; + continue; + } + vars_map[i] = num_cvs++; + if (zend_bitset_in(rc_cvs, i)) { + /* Parameters keep their slot even when they are not refcounted, + * so take the highest refcounted slot rather than the group size. */ + last_var_to_free = vars_map[i] + 1; + } + } + + /* The CVs that can never hold a refcounted value go last, after `last_var_to_free`. */ + for (i = op_array->num_args; i < op_array->last_var; i++) { + if (vars_map[i] == (uint32_t) -1 && zend_bitset_in(used_vars, i)) { + vars_map[i] = num_cvs++; } } + ZEND_ASSERT(last_var_to_free <= num_cvs); - num_tmps = 0; + uint32_t num_tmps = 0; for (i = op_array->last_var; i < op_array->last_var + op_array->T; i++) { if (zend_bitset_in(used_vars, i)) { vars_map[i] = num_cvs + num_tmps++; @@ -71,9 +143,24 @@ void zend_optimizer_compact_vars(zend_op_array *op_array) { } free_alloca(used_vars, use_heap1); + free_alloca(rc_cvs, use_heap3); + + op_array->last_var_to_free = last_var_to_free; + + /* Nothing was removed and no CV moved: the rewrite below would be a no-op. + * Check if anything got moved, as the number of vars could still be the same in that case. */ if (num_cvs == op_array->last_var && num_tmps == op_array->T) { - free_alloca(vars_map, use_heap2); - return; + bool identity = true; + for (i = 0; i < op_array->last_var; i++) { + if (vars_map[i] != (uint32_t) i) { + identity = false; + break; + } + } + if (identity) { + free_alloca(vars_map, use_heap2); + return; + } } ZEND_ASSERT(num_cvs <= op_array->last_var); @@ -93,8 +180,8 @@ void zend_optimizer_compact_vars(zend_op_array *op_array) { } } - /* Update CV name table */ - if (num_cvs != op_array->last_var) { + /* Update CV name table, either because of CVs being removed or being moved. */ + { if (num_cvs) { zend_string **names = safe_emalloc(sizeof(zend_string *), num_cvs, 0); for (i = 0; i < op_array->last_var; i++) { diff --git a/Zend/Optimizer/zend_dump.c b/Zend/Optimizer/zend_dump.c index c6cc5193b2dc..c7683fff2020 100644 --- a/Zend/Optimizer/zend_dump.c +++ b/Zend/Optimizer/zend_dump.c @@ -968,6 +968,9 @@ ZEND_API void zend_dump_op_array(const zend_op_array *op_array, uint32_t dump_fl op_array->last, op_array->num_args); fprintf(stderr, ", vars=%d, tmps=%d", op_array->last_var, op_array->T); + if (op_array->last_var_to_free != (uint32_t) op_array->last_var) { + fprintf(stderr, ", vars_to_free=%d", op_array->last_var_to_free); + } if (ssa) { fprintf(stderr, ", ssa_vars=%d", ssa->vars_count); } diff --git a/Zend/Optimizer/zend_optimizer.c b/Zend/Optimizer/zend_optimizer.c index 6d5b5f541a90..b7243ddfdebb 100644 --- a/Zend/Optimizer/zend_optimizer.c +++ b/Zend/Optimizer/zend_optimizer.c @@ -1159,7 +1159,9 @@ static void zend_optimize(zend_op_array *op_array, if ((ZEND_OPTIMIZER_PASS_13 & ctx->optimization_level) && (!(ZEND_OPTIMIZER_PASS_6 & ctx->optimization_level) || !(ZEND_OPTIMIZER_PASS_7 & ctx->optimization_level))) { - zend_optimizer_compact_vars(op_array); + /* Reached only when the DFA pass did not run, so there is no type + * information to decide which CVs need to be destroyed. */ + zend_optimizer_compact_vars(op_array, NULL); if (ctx->debug_level & ZEND_DUMP_AFTER_PASS_13) { zend_dump_op_array(op_array, 0, "after pass 13", NULL); } @@ -1685,7 +1687,9 @@ ZEND_API void zend_optimize_script(zend_script *script, zend_long optimization_l if (ZEND_OPTIMIZER_PASS_13 & optimization_level) { for (i = 0; i < call_graph.op_arrays_count; i++) { - zend_optimizer_compact_vars(call_graph.op_arrays[i]); + func_info = ZEND_FUNC_INFO(call_graph.op_arrays[i]); + zend_optimizer_compact_vars(call_graph.op_arrays[i], + func_info ? &func_info->ssa : NULL); if (debug_level & ZEND_DUMP_AFTER_PASS_13) { zend_dump_op_array(call_graph.op_arrays[i], 0, "after pass 13", NULL); } diff --git a/Zend/Optimizer/zend_optimizer_internal.h b/Zend/Optimizer/zend_optimizer_internal.h index 6add0f76bc02..320b460b1199 100644 --- a/Zend/Optimizer/zend_optimizer_internal.h +++ b/Zend/Optimizer/zend_optimizer_internal.h @@ -116,7 +116,7 @@ void zend_dfa_optimize_op_array(zend_op_array *op_array, zend_optimizer_ctx *ctx void zend_optimize_temporary_variables(zend_op_array *op_array, zend_optimizer_ctx *ctx); void zend_optimizer_nop_removal(zend_op_array *op_array, zend_optimizer_ctx *ctx); void zend_optimizer_compact_literals(zend_op_array *op_array, zend_optimizer_ctx *ctx); -void zend_optimizer_compact_vars(zend_op_array *op_array); +void zend_optimizer_compact_vars(zend_op_array *op_array, const zend_ssa *ssa); zend_function *zend_optimizer_get_called_func( const zend_script *script, const zend_op_array *op_array, zend_op *opline, bool *is_prototype); uint32_t zend_optimizer_classify_function(const zend_string *name, uint32_t num_args); diff --git a/Zend/zend_compile.h b/Zend/zend_compile.h index 34a91183b2a9..c91bc9aa2984 100644 --- a/Zend/zend_compile.h +++ b/Zend/zend_compile.h @@ -552,6 +552,8 @@ struct _zend_op_array { uint32_t cache_size; /* number of run_time_cache_slots * sizeof(void*) */ int last_var; /* number of CV variables */ uint32_t last; /* number of opcodes */ + /* Number of leading CV slots that have to be destroyed when the call frame is left. */ + uint32_t last_var_to_free; zend_op *opcodes; ZEND_MAP_PTR_DEF(HashTable *, static_variables_ptr); diff --git a/Zend/zend_execute.c b/Zend/zend_execute.c index 14a340ffee37..93c1d1a54682 100644 --- a/Zend/zend_execute.c +++ b/Zend/zend_execute.c @@ -4298,7 +4298,7 @@ ZEND_API void zend_clean_and_cache_symbol_table(zend_array *symbol_table) /* {{{ static zend_always_inline void i_free_compiled_variables(zend_execute_data *execute_data) /* {{{ */ { zval *cv = EX_VAR_NUM(0); - int count = EX(func)->op_array.last_var; + uint32_t count = EX(func)->op_array.last_var_to_free; while (EXPECTED(count != 0)) { i_zval_ptr_dtor(cv); cv++; diff --git a/Zend/zend_object_handlers.c b/Zend/zend_object_handlers.c index 8ca6d212fd72..1142260ccd2c 100644 --- a/Zend/zend_object_handlers.c +++ b/Zend/zend_object_handlers.c @@ -1827,6 +1827,7 @@ ZEND_API ZEND_ATTRIBUTE_NONNULL zend_function *zend_get_call_trampoline_func( * value so that it doesn't contain garbage when the engine allocates space for the next stack * frame. This didn't cause any issues until now due to "lucky" structure layout. */ func->last_var = 0; + func->last_var_to_free = 0; uint32_t min_T = 2 + ZEND_OBSERVER_ENABLED; func->T = (fbc->type == ZEND_USER_FUNCTION)? MAX(fbc->op_array.last_var + fbc->op_array.T, min_T) : min_T; func->filename = (fbc->type == ZEND_USER_FUNCTION)? fbc->op_array.filename : ZSTR_EMPTY_ALLOC(); diff --git a/Zend/zend_opcode.c b/Zend/zend_opcode.c index a25f236c3c20..675c660a6e39 100644 --- a/Zend/zend_opcode.c +++ b/Zend/zend_opcode.c @@ -57,6 +57,7 @@ void init_op_array(zend_op_array *op_array, zend_function_type type, int initial op_array->opcodes = emalloc(initial_ops_size * sizeof(zend_op)); op_array->last_var = 0; + op_array->last_var_to_free = 0; op_array->vars = NULL; op_array->T = 0; @@ -1120,6 +1121,9 @@ ZEND_API void pass_two(zend_op_array *op_array) CG(context).vars_size = op_array->last_var; } + /* Without type inference every CV has to be assumed refcounted; this can be lowered via the optimizer. */ + op_array->last_var_to_free = op_array->last_var; + #if ZEND_USE_ABS_CONST_ADDR if (CG(context).opcodes_size != op_array->last) { op_array->opcodes = (zend_op *) erealloc(op_array->opcodes, sizeof(zend_op)*op_array->last); diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index e91da6aeb8d3..079cdabc4fca 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -2930,7 +2930,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op zend_jit_common_return(jit); bool left_frame = false; - if (op_array->last_var > 100) { + if (op_array->last_var_to_free > 100) { /* To many CVs to unroll */ if (!zend_jit_free_cvs(&ctx)) { goto jit_failure; @@ -2938,9 +2938,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op left_frame = true; } if (!left_frame) { - int j; - - for (j = 0 ; j < op_array->last_var; j++) { + for (uint32_t j = 0; j < op_array->last_var_to_free; j++) { uint32_t info = zend_ssa_cv_info(op_array, ssa, j); if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index a47ef18db337..8a0bf589f516 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -5597,7 +5597,6 @@ static zend_vm_opcode_handler_t zend_jit_trace(zend_jit_trace_rec *trace_buffer, goto jit_failure; } } else { - int j; int may_throw = 0; bool left_frame = 0; @@ -5605,7 +5604,7 @@ static zend_vm_opcode_handler_t zend_jit_trace(zend_jit_trace_rec *trace_buffer, op1_info, OP1_REG_ADDR())) { goto jit_failure; } - if (op_array->last_var > 100) { + if (op_array->last_var_to_free > 100) { /* To many CVs to unroll */ if (!zend_jit_free_cvs(&ctx)) { goto jit_failure; @@ -5613,7 +5612,7 @@ static zend_vm_opcode_handler_t zend_jit_trace(zend_jit_trace_rec *trace_buffer, left_frame = 1; } if (!left_frame) { - for (j = 0 ; j < op_array->last_var; j++) { + for (uint32_t j = 0; j < op_array->last_var_to_free; j++) { uint32_t info; uint8_t type; diff --git a/ext/opcache/tests/opt/coalesce.phpt b/ext/opcache/tests/opt/coalesce.phpt index f60294aadc23..18f12724ea44 100644 --- a/ext/opcache/tests/opt/coalesce.phpt +++ b/ext/opcache/tests/opt/coalesce.phpt @@ -30,14 +30,14 @@ $_main: 0000 RETURN int(1) a: - ; (lines=2, args=0, vars=1, tmps=1) + ; (lines=2, args=0, vars=1, tmps=1, vars_to_free=0) ; (after optimizer) ; %s 0000 T1 = COALESCE CV0($test) 0001 0001 RETURN bool(true) b: - ; (lines=2, args=0, vars=1, tmps=1) + ; (lines=2, args=0, vars=1, tmps=1, vars_to_free=0) ; (after optimizer) ; %s 0000 T1 = COALESCE CV0($test) 0001 diff --git a/ext/opcache/tests/opt/sccp_022.phpt b/ext/opcache/tests/opt/sccp_022.phpt index 27d7b6d71928..2a468096707e 100644 --- a/ext/opcache/tests/opt/sccp_022.phpt +++ b/ext/opcache/tests/opt/sccp_022.phpt @@ -48,7 +48,7 @@ foo: 0010 RETURN null bar: - ; (lines=9, args=0, vars=3, tmps=2) + ; (lines=9, args=0, vars=3, tmps=2, vars_to_free=0) ; (after optimizer) ; %s 0000 T3 = FE_RESET_R CV0($a) 0007 diff --git a/ext/opcache/tests/opt/sccp_024.phpt b/ext/opcache/tests/opt/sccp_024.phpt index 56be429cc799..ec5c13f7ce8a 100644 --- a/ext/opcache/tests/opt/sccp_024.phpt +++ b/ext/opcache/tests/opt/sccp_024.phpt @@ -30,7 +30,7 @@ $_main: 0000 RETURN int(1) A::t: - ; (lines=10, args=1, vars=2, tmps=2) + ; (lines=10, args=1, vars=2, tmps=2, vars_to_free=1) ; (after optimizer) ; %ssccp_024.php:3-10 0000 CV0($obj) = RECV 1 diff --git a/ext/opcache/tests/opt/sccp_026.phpt b/ext/opcache/tests/opt/sccp_026.phpt index 3e318a7f75df..043e8ffe1aa2 100644 --- a/ext/opcache/tests/opt/sccp_026.phpt +++ b/ext/opcache/tests/opt/sccp_026.phpt @@ -27,7 +27,7 @@ $_main: 0000 RETURN int(1) test: - ; (lines=9, args=1, vars=2, tmps=1) + ; (lines=9, args=1, vars=2, tmps=1, vars_to_free=1) ; (after optimizer) ; %s:2-8 0000 CV0($var) = RECV 1