diff --git a/.github/workflows/zjit-macos.yml b/.github/workflows/zjit-macos.yml index 82e1d911eb5e62..4e574b6be167f8 100644 --- a/.github/workflows/zjit-macos.yml +++ b/.github/workflows/zjit-macos.yml @@ -98,7 +98,7 @@ jobs: rustup install ${{ matrix.rust_version }} --profile minimal rustup default ${{ matrix.rust_version }} - - uses: taiki-e/install-action@288e746965032cfcc232e09af2daf5f23c14d780 # v2.86.1 + - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 with: tool: nextest@0.9 if: ${{ matrix.test_task == 'zjit-check' }} diff --git a/.github/workflows/zjit-ubuntu.yml b/.github/workflows/zjit-ubuntu.yml index c14fceb2437092..2f323a61f4a78b 100644 --- a/.github/workflows/zjit-ubuntu.yml +++ b/.github/workflows/zjit-ubuntu.yml @@ -152,7 +152,7 @@ jobs: ruby-version: '3.1' bundler: none - - uses: taiki-e/install-action@288e746965032cfcc232e09af2daf5f23c14d780 # v2.86.1 + - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 with: tool: nextest@0.9 if: ${{ matrix.test_task == 'zjit-check' }} diff --git a/compile.c b/compile.c index 5cc888bcc39cf2..95f49b53a9ea27 100644 --- a/compile.c +++ b/compile.c @@ -1864,14 +1864,14 @@ update_lvar_state(const rb_iseq_t *iseq, int level, int idx) iseq = ISEQ_BODY(iseq)->parent_iseq; } - enum lvar_state *states = ISEQ_BODY(iseq)->lvar_states; + uint8_t *states = ISEQ_BODY(iseq)->lvar_states; int table_idx = ISEQ_BODY(iseq)->local_table_size - idx; - switch (states[table_idx]) { + switch (iseq_lvar_state_get(states, table_idx)) { case lvar_uninitialized: - states[table_idx] = lvar_initialized; + iseq_lvar_state_set(states, table_idx, lvar_initialized); break; case lvar_initialized: - states[table_idx] = lvar_reassigned; + iseq_lvar_state_set(states, table_idx, lvar_reassigned); break; case lvar_reassigned: /* nothing */ @@ -1885,13 +1885,13 @@ static int iseq_set_parameters_lvar_state(const rb_iseq_t *iseq) { for (unsigned int i=0; iparam.size; i++) { - ISEQ_BODY(iseq)->lvar_states[i] = lvar_initialized; + iseq_lvar_state_set(ISEQ_BODY(iseq)->lvar_states, i, lvar_initialized); } int lead_num = ISEQ_BODY(iseq)->param.lead_num; int opt_num = ISEQ_BODY(iseq)->param.opt_num; for (int i=0; ilvar_states[lead_num + i] = lvar_uninitialized; + iseq_lvar_state_set(ISEQ_BODY(iseq)->lvar_states, lead_num + i, lvar_uninitialized); } return COMPILE_OK; @@ -2257,13 +2257,7 @@ iseq_set_local_table(rb_iseq_t *iseq, const rb_ast_id_table_t *tbl, const NODE * MEMCPY(ids, tbl->ids + offset, ID, size); ISEQ_BODY(iseq)->local_table = ids; - enum lvar_state *states = ALLOC_N(enum lvar_state, size); - // fprintf(stderr, "iseq:%p states:%p size:%d\n", iseq, states, (int)size); - for (unsigned int i=0; ilocal_table[i])); - } - ISEQ_BODY(iseq)->lvar_states = states; + ISEQ_BODY(iseq)->lvar_states = ZALLOC_N(uint8_t, ISEQ_LVAR_STATES_BUFLEN(size)); } ISEQ_BODY(iseq)->local_table_size = size; @@ -12616,7 +12610,7 @@ typedef uint32_t ibf_offset_t; #define IBF_MAJOR_VERSION ISEQ_MAJOR_VERSION #ifdef RUBY_DEVEL -#define IBF_DEVEL_VERSION 6 +#define IBF_DEVEL_VERSION 7 #define IBF_MINOR_VERSION (ISEQ_MINOR_VERSION * 10000 + IBF_DEVEL_VERSION) #else #define IBF_MINOR_VERSION ISEQ_MINOR_VERSION @@ -13443,12 +13437,12 @@ static ibf_offset_t ibf_dump_lvar_states(struct ibf_dump *dump, const rb_iseq_t *iseq) { const struct rb_iseq_constant_body *const body = ISEQ_BODY(iseq); - const int size = body->local_table_size; - IBF_W_ALIGN(enum lvar_state); - return ibf_dump_write(dump, body->lvar_states, sizeof(enum lvar_state) * (body->lvar_states ? size : 0)); + const int size = ISEQ_LVAR_STATES_BUFLEN(body->local_table_size); + IBF_W_ALIGN(uint8_t); + return ibf_dump_write(dump, body->lvar_states, sizeof(uint8_t) * (body->lvar_states ? size : 0)); } -static enum lvar_state * +static uint8_t * ibf_load_lvar_states(const struct ibf_load *load, ibf_offset_t lvar_states_offset, int size, const ID *local_table) { if (local_table == rb_iseq_shared_exc_local_tbl || @@ -13456,7 +13450,7 @@ ibf_load_lvar_states(const struct ibf_load *load, ibf_offset_t lvar_states_offse return NULL; } else { - enum lvar_state *states = IBF_R(lvar_states_offset, enum lvar_state, size); + uint8_t *states = IBF_R(lvar_states_offset, uint8_t, ISEQ_LVAR_STATES_BUFLEN(size)); return states; } } diff --git a/gc.c b/gc.c index 9e7410a404e264..f72d2c09af374c 100644 --- a/gc.c +++ b/gc.c @@ -686,7 +686,6 @@ typedef struct gc_function_map { void (*writebarrier_unprotect)(void *objspace_ptr, VALUE obj); void (*writebarrier_remember)(void *objspace_ptr, VALUE obj); void (*obj_became_shareable)(void *objspace_ptr, VALUE obj); - void (*pin_in_flight_message)(void *objspace_ptr, VALUE obj); // Heap walking void (*each_objects)(void *objspace_ptr, int (*callback)(void *, void *, size_t, void *), void *data); void (*each_objects_shareable)(void *objspace_ptr, int (*callback)(void *, void *, size_t, void *), void *data); @@ -882,7 +881,6 @@ ruby_modular_gc_init(void) load_modular_gc_func(writebarrier_unprotect); load_modular_gc_func(writebarrier_remember); load_modular_gc_func(obj_became_shareable); - load_modular_gc_func(pin_in_flight_message); // Heap walking load_modular_gc_func(each_objects); load_modular_gc_func(each_objects_shareable); @@ -987,7 +985,6 @@ ruby_modular_gc_init(void) # define rb_gc_impl_writebarrier_unprotect rb_gc_functions.writebarrier_unprotect # define rb_gc_impl_writebarrier_remember rb_gc_functions.writebarrier_remember # define rb_gc_impl_obj_became_shareable rb_gc_functions.obj_became_shareable -# define rb_gc_impl_pin_in_flight_message rb_gc_functions.pin_in_flight_message // Heap walking # define rb_gc_impl_each_objects rb_gc_functions.each_objects # define rb_gc_impl_each_objects_shareable rb_gc_functions.each_objects_shareable @@ -3267,14 +3264,12 @@ rb_gc_mark_roots(void *objspace, const char **categoryp) !rb_gc_impl_multi_objspace_p(); /* Mark the current Ractor's roots from its C structs (a local GC must not depend on - * heap wrapper traversal). A global GC does the same for every Ractor and re-pins - * the in-flight payloads whose shrefs its clear pass dropped. */ + * heap wrapper traversal). A global GC does the same for every Ractor. */ MARK_CHECKPOINT("ractor"); if (global_gc) { rb_ractor_t *r; ccan_list_for_each(&vm->ractor.set, r, vmlr_node) { rb_ractor_mark_local_roots(r); - rb_ractor_repin_in_flight(r); } /* Early in boot (before rb_ractor_main_setup) main is not in vm->ractor.set @@ -3284,15 +3279,13 @@ rb_gc_mark_roots(void *objspace, const char **categoryp) } /* A Ractor that terminated (left vm->ractor.set) but whose struct is not freed * still owns rb_gc_register_mark_object pins. Keep them alive until - * ractor_free hands them to main; an orphan (owner == NULL) was moved above. */ + * ractor_free hands them to main; an orphan (owner == NULL) was moved above. + * The join value is not rooted here: ractor_mark marks it from the wrapper. */ for (size_t i = 0; i < vm->gc.zombie_objspaces_count; i++) { rb_ractor_t *owner = vm->gc.zombie_objspaces[i].owner; if (owner) { rb_gc_mark_vm_stack_values((long)owner->registered_marks_cnt, owner->registered_marks); - /* Keep a terminated Ractor's join value (read by Ractor#value) alive - * without depending on wrapper reachability. Threads are not walked. */ - rb_ractor_mark_terminated_join_value(owner); } } @@ -3346,15 +3339,6 @@ rb_gc_mark_roots(void *objspace, const char **categoryp) if (vm_mark_needs_lock) vm_mark_lock_lev = RB_GC_VM_LOCK_NO_BARRIER(); rb_vm_mark(vm); - if (global_gc) { - /* Mark and pin the shareable REFs of in-flight (off-heap) move couriers, - * covering the transient window between queue and materialize frame. Only - * a global GC frees shareable objects, so only it needs this pass. */ - MARK_CHECKPOINT("move_couriers"); - void rb_ractor_move_courier_registry_mark(void); - rb_ractor_move_courier_registry_mark(); - } - MARK_CHECKPOINT("global_tbl"); rb_gc_mark_global_tbl(); @@ -3757,14 +3741,6 @@ rb_gc_obj_became_shareable(VALUE obj) /* Pin an in-flight message payload in its owner's (the sender's) objspace, so the * sender's local GC keeps it alive while it sits in a queue the sender does not walk. */ -void -rb_gc_pin_in_flight_message(VALUE obj) -{ - if (RB_SPECIAL_CONST_P(obj)) return; - - rb_gc_impl_pin_in_flight_message(rb_gc_get_objspace(), obj); -} - void rb_gc_copy_attributes(VALUE dest, VALUE obj) { @@ -4159,11 +4135,8 @@ rb_gc_vm_refresh_zombie_pages(void) vm->gc.zombie_total_pages = total; } -/* Incremental marking only runs single-objspace; vm_insert_ractor0 calls this just - * before a second Ractor becomes visible so any cycle in progress finishes; a settle - * cannot resume, nor inheritance extend, another objspace's partial mark. */ void -rb_gc_finish_in_flight_gc(void) +rb_gc_rest(void) { rb_gc_impl_gc_rest(rb_gc_get_objspace()); } @@ -4954,13 +4927,6 @@ rb_gc_vm_generic_fields_drain_dead(bool (*is_dead)(VALUE key)) rb_generic_fields_tables_foreach(gf_drain_table_cb, &ctx); } -/* A wrapper exported from gc.c so a modular build's gc-impl can call it. */ -bool -rb_gc_current_ractor_materializing_p(void) -{ - return rb_ractor_materializing_p(); -} - VALUE rb_gc_vm_top_self(void) { diff --git a/gc/default/default.c b/gc/default/default.c index cefcf9d508838c..a0e2cfd0a64206 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -6415,15 +6415,13 @@ check_children_i(const VALUE child, void *ptr) * unshareable parent holding an unrecorded foreign unshareable child would be * invisible to both local GCs. The exception is a box's top_self, which every * thread's th->top_self points at and which is VM-permanent. Skipped during a - * global GC: it clears every shref bit and keeps in-flight payloads alive by - * re-pinning, so the shref exemption would not fire, and its unified exact - * stop-the-world mark makes the invariant itself moot. */ + * global GC: it clears every shref bit, so the shref exemption would not fire, + * and its unified exact stop-the-world mark makes the invariant itself moot. */ if (!data->parent_shareable && child != rb_gc_vm_top_self() && !MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(child), child) && !MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(child), child) && !rb_gc_impl_during_global_gc_p(data->objspace) && - !rb_gc_current_ractor_materializing_p() && !global_objspace->during_absorb) { fprintf(stderr, "check_children_i: containment violation: " "unshareable %s (objspace %p) -> foreign unshareable %s (objspace %p)\n", @@ -6500,10 +6498,6 @@ root_scope_check_i(const char *category, VALUE obj, void *ptr) if (MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(obj), obj)) return; if (MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(obj), obj)) return; if (obj == rb_gc_vm_top_self()) return; /* VM-permanent (see check_children_i) */ - /* A sender-resident snapshot being materialized by a receive is rooted through - * sync.materializing_copies: a foreign-unshareable root that is valid only while - * the copy runs (see check_children_i). */ - if (rb_gc_current_ractor_materializing_p()) return; fprintf(stderr, "root_scope_check_i: root category \"%s\" names a foreign " "unshareable without a shref record: %s (owner %p, self %p)\n", @@ -7843,26 +7837,6 @@ rb_gc_impl_obj_became_shareable(void *objspace_ptr, VALUE obj) } } -void -rb_gc_impl_pin_in_flight_message(void *objspace_ptr, VALUE obj) -{ - if (RB_FL_TEST_RAW(obj, RUBY_FL_SHAREABLE)) return; /* pinned anyway */ - - /* The payload's pages belong to the sender, so a plain store is enough. */ - struct heap_page *page = GET_HEAP_PAGE(obj); - if (!_MARKED_IN_BITMAP(page->shref_bits, page, obj)) { - _MARK_IN_BITMAP(page->shref_bits, page, obj); - page->flags.has_shref_objects = TRUE; - } - /* A shref bit only makes the object a root for the next local GC; it does not affect an - * in-progress global compaction's move decision (pinned_bits). Moving a payload node - * would break the address-keyed maps, dedup tables and pin lists, so pin it as well. */ - rb_objspace_t *objspace = objspace_ptr; - if (objspace->flags.during_global_gc) { - gc_pin(objspace, obj); - } -} - void rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj) { @@ -9069,8 +9043,8 @@ gc_start_global(rb_objspace_t *driver, unsigned int reason, bool compact, bool a } } - /* steps 6-7: every Ractor's roots (gc.c walks them all and re-pins in-flight payloads), - * then one unified precise mark. A global GC does not go through gc_marks, so the marking + /* steps 6-7: every Ractor's roots (gc.c walks them all), then one unified precise + * mark. A global GC does not go through gc_marks, so the marking * phase is opened here instead; it closes after rb_ractor_finish_marking below, which is * where gc_marks_finish ends for a local collection. */ gc_marking_enter(driver); diff --git a/gc/gc.h b/gc/gc.h index 0ff503a222df3e..656f403d8c4d79 100644 --- a/gc/gc.h +++ b/gc/gc.h @@ -90,7 +90,6 @@ MODULAR_GC_FN void rb_gc_vm_weak_table_foreach(vm_table_foreach_callback_func ca MODULAR_GC_FN void rb_gc_vm_generic_fields_mark_foreach(int (*cb)(VALUE key, VALUE val, void *arg), void *arg); MODULAR_GC_FN void rb_gc_vm_generic_fields_drain_dead(bool (*is_dead)(VALUE key)); /* Exemptions for the shareable containment verifier (called from a gc-impl). */ -MODULAR_GC_FN bool rb_gc_current_ractor_materializing_p(void); MODULAR_GC_FN VALUE rb_gc_vm_top_self(void); MODULAR_GC_FN void rb_gc_update_object_references(void *objspace, VALUE obj); MODULAR_GC_FN void rb_gc_update_vm_references(void *objspace); diff --git a/gc/gc_impl.h b/gc/gc_impl.h index ef5bc5a5f2894c..76634e62992664 100644 --- a/gc/gc_impl.h +++ b/gc/gc_impl.h @@ -129,7 +129,6 @@ GC_IMPL_FN void rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b); GC_IMPL_FN void rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj); GC_IMPL_FN void rb_gc_impl_writebarrier_remember(void *objspace_ptr, VALUE obj); GC_IMPL_FN void rb_gc_impl_obj_became_shareable(void *objspace_ptr, VALUE obj); -GC_IMPL_FN void rb_gc_impl_pin_in_flight_message(void *objspace_ptr, VALUE obj); // Heap walking GC_IMPL_FN void rb_gc_impl_each_objects(void *objspace_ptr, int (*callback)(void *, void *, size_t, void *), void *data); GC_IMPL_FN void rb_gc_impl_each_objects_shareable(void *objspace_ptr, int (*callback)(void *, void *, size_t, void *), void *data); diff --git a/gc/mmtk/mmtk.c b/gc/mmtk/mmtk.c index a7aec6e68338e7..8e0c7a72e88e69 100644 --- a/gc/mmtk/mmtk.c +++ b/gc/mmtk/mmtk.c @@ -1265,12 +1265,6 @@ rb_gc_impl_obj_became_shareable(void *objspace_ptr, VALUE obj) /* MMTk has no per-page shareable bits. */ } -void -rb_gc_impl_pin_in_flight_message(void *objspace_ptr, VALUE obj) -{ - /* With a single objspace there is nothing to pin. */ -} - void rb_gc_impl_writebarrier_remember(void *objspace_ptr, VALUE obj) { diff --git a/internal/gc.h b/internal/gc.h index 23392d36230e66..5bb87ef1543907 100644 --- a/internal/gc.h +++ b/internal/gc.h @@ -299,7 +299,6 @@ rb_obj_atomic_write( int rb_ec_stack_check(struct rb_execution_context_struct *ec); void rb_gc_writebarrier_remember(VALUE obj); void rb_gc_obj_became_shareable(VALUE obj); -void rb_gc_pin_in_flight_message(VALUE obj); bool rb_gc_multi_objspace_p(void); bool rb_gc_obj_foreign_p(VALUE obj); void *rb_gc_objspace_alloc(void); @@ -313,7 +312,7 @@ void rb_gc_zombie_objspaces_atfork(void); void rb_gc_disable_holders_atfork(void); void rb_gc_atfork_global_locks(void); void rb_gc_stash_cleanup_objspace(void); -void rb_gc_finish_in_flight_gc(void); +void rb_gc_rest(void); bool rb_gc_during_global_gc_p(void); bool rb_gc_single_objspace_p(void); const char *rb_obj_info(VALUE obj); diff --git a/internal/re.h b/internal/re.h index 0d4bc43ad40acb..49d82d5c209727 100644 --- a/internal/re.h +++ b/internal/re.h @@ -65,11 +65,11 @@ long rb_reg_search0(VALUE, VALUE, long, int, int, VALUE *); VALUE rb_reg_match_p(VALUE re, VALUE str, long pos); VALUE rb_reg_regsub_match(VALUE str, VALUE src, VALUE match); VALUE rb_match_init_copy(VALUE copy, VALUE orig); -/* MatchData transfer for the move courier (ractor.c). */ -void *rb_match_move_dump(VALUE match, VALUE *regexp_out, VALUE *str_out, int *num_regs_out); -VALUE rb_match_move_alloc(VALUE klass, int num_regs); -void rb_match_move_load(VALUE match, VALUE regexp, VALUE str, int num_regs, const void *blob); -void rb_match_move_free(void *blob); +/* MatchData transfer for the Ractor courier (ractor.c). */ +void *rb_match_blob_dump(VALUE match, VALUE *regexp_out, VALUE *str_out, int *num_regs_out, bool release_source); +VALUE rb_match_blob_alloc(VALUE klass, int num_regs); +void rb_match_blob_load(VALUE match, VALUE regexp, VALUE str, int num_regs, const void *blob); +void rb_match_blob_free(void *blob); bool rb_reg_start_with_p(VALUE re, VALUE str); VALUE rb_reg_hash(VALUE re); VALUE rb_reg_equal(VALUE re1, VALUE re2); diff --git a/internal/string.h b/internal/string.h index a8893a42f1e10a..67216f574879de 100644 --- a/internal/string.h +++ b/internal/string.h @@ -124,6 +124,7 @@ bool rb_obj_is_fstring_table(VALUE obj); void Init_fstring_table(); VALUE rb_obj_as_string_result(VALUE str, VALUE obj); VALUE rb_str_opt_plus(VALUE x, VALUE y); +VALUE rb_str_new_owned(char *ptr, long len, long capa, int encindex); VALUE rb_str_concat_literals(size_t num, const VALUE *strary); VALUE rb_str_eql(VALUE str1, VALUE str2); VALUE rb_id_quote_unprintable(ID); diff --git a/internal/vm.h b/internal/vm.h index 1820a4e69f2d2a..560c51d703435d 100644 --- a/internal/vm.h +++ b/internal/vm.h @@ -115,6 +115,9 @@ VALUE rb_make_backtrace(void); void rb_backtrace_print_as_bugreport(FILE*); int rb_backtrace_p(VALUE obj); VALUE rb_backtrace_dup(VALUE btobj); +void *rb_backtrace_blob_dump(VALUE btobj, int *size_out); +VALUE rb_backtrace_blob_load(const void *blob, int size); +void rb_backtrace_blob_mark(const void *blob, int size); VALUE rb_backtrace_to_str_ary(VALUE obj); VALUE rb_backtrace_to_location_ary(VALUE obj); VALUE rb_location_ary_to_backtrace(VALUE ary); diff --git a/iseq.c b/iseq.c index 5a12633dd78b1c..e1cc8a364bb6ae 100644 --- a/iseq.c +++ b/iseq.c @@ -231,7 +231,7 @@ rb_iseq_free(const rb_iseq_t *iseq) if (LIKELY(body->local_table != rb_iseq_shared_exc_local_tbl)) { SIZED_FREE_N(body->local_table, body->local_table_size); } - SIZED_FREE_N(body->lvar_states, body->local_table_size); + SIZED_FREE_N(body->lvar_states, ISEQ_LVAR_STATES_BUFLEN(body->local_table_size)); compile_data_free(ISEQ_COMPILE_DATA(iseq)); if (body->outer_variables) rb_id_table_free(body->outer_variables); @@ -544,7 +544,7 @@ rb_iseq_memsize(const rb_iseq_t *iseq) size += body->iseq_size * sizeof(VALUE); size += body->insns_info.size * (sizeof(struct iseq_insn_info_entry) + sizeof(unsigned int)); size += body->local_table_size * sizeof(ID); // body->local_table - if (body->lvar_states) size += body->local_table_size * sizeof(enum lvar_state); + if (body->lvar_states) size += ISEQ_LVAR_STATES_BUFLEN(body->local_table_size) * sizeof(uint8_t); size += ISEQ_MBITS_BUFLEN(body->iseq_size) * ISEQ_MBITS_SIZE; if (body->catch_table) { size += iseq_catch_table_bytes(body->catch_table->size); diff --git a/iseq.h b/iseq.h index c9bdfcb484759f..e641dd6aed1f5e 100644 --- a/iseq.h +++ b/iseq.h @@ -26,6 +26,26 @@ RUBY_EXTERN const int ruby_api_version[]; #define ISEQ_MBITS_SET_P(buf, i) ((buf[(i) / ISEQ_MBITS_BITLENGTH] >> ((i) % ISEQ_MBITS_BITLENGTH)) & 0x1) #define ISEQ_MBITS_BUFLEN(size) roomof(size, ISEQ_MBITS_BITLENGTH) +#define ISEQ_LVAR_STATE_BITS 2 +#define ISEQ_LVAR_STATES_PER_BYTE (CHAR_BIT / ISEQ_LVAR_STATE_BITS) +#define ISEQ_LVAR_STATES_BUFLEN(size) roomof(size, ISEQ_LVAR_STATES_PER_BYTE) +STATIC_ASSERT(lvar_state_fits_in_iseq_lvar_state_bits, lvar_reassigned < (1 << ISEQ_LVAR_STATE_BITS)); + +static inline enum lvar_state +iseq_lvar_state_get(const uint8_t *buf, unsigned int i) +{ + const unsigned int shift = (i % ISEQ_LVAR_STATES_PER_BYTE) * ISEQ_LVAR_STATE_BITS; + return (enum lvar_state)((buf[i / ISEQ_LVAR_STATES_PER_BYTE] >> shift) & ((1 << ISEQ_LVAR_STATE_BITS) - 1)); +} + +static inline void +iseq_lvar_state_set(uint8_t *buf, unsigned int i, enum lvar_state state) +{ + uint8_t *const byte = &buf[i / ISEQ_LVAR_STATES_PER_BYTE]; + const unsigned int shift = (i % ISEQ_LVAR_STATES_PER_BYTE) * ISEQ_LVAR_STATE_BITS; + *byte = (*byte & ~(((1 << ISEQ_LVAR_STATE_BITS) - 1) << shift)) | ((uint8_t)state << shift); +} + #ifndef USE_ISEQ_NODE_ID #define USE_ISEQ_NODE_ID 1 #endif diff --git a/ractor.c b/ractor.c index c6b6c32b8242b6..a26176619a15b2 100644 --- a/ractor.c +++ b/ractor.c @@ -306,11 +306,18 @@ ractor_mark(void *ptr) * both the set and zombie_objspaces (orphan-merged) this marker is its only cover. */ rb_gc_mark(r->sync.default_port_value); /* A single-objspace impl (mmtk) has no zombie_objspaces and no pin/shref bits, so - * the root scan cannot reach a terminated Ractor's legacy value, queue or in-flight - * payloads; and no shref rule forbids following them from the wrapper. */ + * the root scan cannot reach a terminated Ractor's queue, in-flight payloads or + * join value; and no shref rule forbids following them from the wrapper. */ if (!rb_gc_multi_objspace_p()) { ractor_mark_unshareable_parts(r); - rb_ractor_mark_in_flight_for_single_objspace(r); + rb_ractor_mark_terminated_join_value(r); + } + else if (rb_gc_during_global_gc_p()) { + /* The join value is only of use to whoever can still call Ractor#value, which + * means holding this wrapper, so mark it as the wrapper's child rather than as a + * root. A global GC stops the world and marks every objspace together, which is + * what lets the shareable wrapper reach an unshareable value at all. */ + rb_ractor_mark_terminated_join_value(r); } } @@ -442,10 +449,6 @@ ractor_free(void *ptr) r->registered_marks = NULL; r->registered_marks_cnt = r->registered_marks_capa = 0; - free(r->pin_capture); - r->pin_capture = NULL; - r->pin_capture_cnt = r->pin_capture_capa = 0; - if (!r->main_ractor) { SIZED_FREE(r); } @@ -520,10 +523,11 @@ vm_insert_ractor0(rb_vm_t *vm, rb_ractor_t *r, bool single_ractor_mode) RUBY_DEBUG_LOG("r:%u ractor.cnt:%u++", r->pub.id, vm->ractor.cnt); VM_ASSERT(single_ractor_mode || RB_VM_LOCKED_P()); - /* Just before the process goes multi-objspace. Incremental marking only runs in a - * single-objspace world, so finish any cycle in progress before the count changes. */ + /* Incremental marking only runs in a single-objspace world, and nothing later can + * finish another objspace's partial mark, so end any cycle in progress before a + * second Ractor becomes visible. */ if (vm->ractor.cnt == 1) { - rb_gc_finish_in_flight_gc(); + rb_gc_rest(); } ccan_list_add_tail(&vm->ractor.set, &r->vmlr_node); @@ -727,10 +731,6 @@ rb_ractor_atfork(rb_vm_t *vm, rb_thread_t *th) // initialize as a main ractor vm->ractor.cnt = 0; vm->ractor.blocking_cnt = 0; - /* Another thread may have held the lock at fork, so rebuild it in the child (the - * same reason generic_fields_lock is re-initialized at fork). The registry's list - * head is left alone: the nodes of surviving couriers are still linked into it. */ - rb_native_mutex_initialize(&vm->ractor.move_courier_registry_lock); /* Only main survives a fork: the holds of dead Ractors and of critical sections are * gone, leaving main's own disable. */ rb_gc_disable_holders_atfork(); @@ -777,10 +777,6 @@ static void ractor_init(rb_ractor_t *r, VALUE name, VALUE loc) { ractor_sync_init(r); - r->gen_fields_capturing = false; - r->pin_capture = NULL; - r->pin_capture_cnt = r->pin_capture_capa = 0; - r->sending_basket = NULL; st_init_existing_numtable_with_size(&r->pub.targeted_hooks, 0); r->pub.hooks.type = hook_list_type_ractor_local; @@ -2369,24 +2365,26 @@ rb_obj_traverse_replace(VALUE obj, } } -/* Move courier: serializes the payload of Ractor#send(move: true) into an xmalloc'd +/* Courier: serializes a Ractor message payload -- copied or moved -- into an xmalloc'd * structure that belongs to no objspace, so no sender GC can mark, sweep, compact or * race with it. A node array with id references handles sharing and cycles, and the - * receiver rebuilds it in its own objspace in two passes. */ - -enum move_node_kind { - MOVE_KIND_REF, /* an immediate or a shareable object: carried by value */ - MOVE_KIND_STRING, - MOVE_KIND_ARRAY, - MOVE_KIND_HASH, - MOVE_KIND_OBJECT, - MOVE_KIND_STRUCT, - MOVE_KIND_MATCH, - MOVE_KIND_IO, + * receiver rebuilds it in its own objspace in two passes. Copy and move differ only in + * whether the source is read or taken apart: see courier_build.copy. */ + +enum courier_node_kind { + COURIER_KIND_REF, /* an immediate or a shareable object: carried by value */ + COURIER_KIND_BACKTRACE, /* an exception's backtrace: frames copied into an off-heap blob */ + COURIER_KIND_STRING, + COURIER_KIND_ARRAY, + COURIER_KIND_HASH, + COURIER_KIND_OBJECT, + COURIER_KIND_STRUCT, + COURIER_KIND_MATCH, + COURIER_KIND_IO, }; -struct move_node { - enum move_node_kind kind; +struct courier_node { + enum courier_node_kind kind; bool frozen; /* The instance and generic ivars every non-REF node can have (a String or Array * can hold generic ivars too) */ @@ -2395,12 +2393,13 @@ struct move_node { uint32_t *iv_vals; /* owned by the courier; node ids */ union { VALUE ref; - struct { char *ptr; long len; int encidx; VALUE klass; } str; /* the courier owns ptr */ + struct { char *ptr; long len, capa; int encidx; VALUE klass; } str; /* the courier owns ptr */ struct { long len; uint32_t *elems; VALUE klass; } ary; /* the courier owns elems */ struct { long size; uint32_t *kv; uint32_t ifnone_id; bool compare_by_id; bool proc_default; VALUE klass; } hash; /* owns kv (2*size) */ struct { VALUE klass; } obj; struct { long len; uint32_t *elems; VALUE klass; } strct; /* owns elems */ struct { uint32_t regexp_id, str_id; int num_regs; void *regs; VALUE klass; } match; /* owns regs */ + struct { void *blob; int size; } bt; /* the courier owns blob */ struct { struct rb_io *fptr; /* carried by pointer (it owns the fd) */ VALUE klass; @@ -2412,75 +2411,113 @@ struct move_node { } u; }; -struct rb_ractor_move_courier { - struct move_node *nodes; +/* A child slot holds a node id, or -- with this bit set -- an index into c->refs. + * The courier is in-process, so a shareable payload can travel as the VALUE itself + * instead of costing a whole courier_node; the basket holding the courier marks c->refs. */ +#define COURIER_ID_REF_BIT 0x80000000u + +struct rb_ractor_courier { + struct courier_node *nodes; uint32_t count; uint32_t capa; + VALUE *refs; /* shareable payloads, embedded by value */ + uint32_t refs_count; + uint32_t refs_capa; uint32_t root; - struct ccan_list_node reg_node; /* in-flight courier registry (a GC root while it lives) */ }; -/* VM-global list of move couriers in flight (vm->ractor.move_courier_registry). A - * courier is off-heap and carries shareable REFs as raw pointers; in some windows only - * a transient (a stack-local message queue, say) reaches it, so a global GC could - * collect the REFs. Registered from build to free, marked and pinned by the global - * GC's root pass (only a global GC frees shareable objects, so only it needs this). - * add/remove run concurrently and take the lock; stop-the-world marking does not, which - * is sound only because add/remove contain no safepoint (none may be added: a mark - * could then see a half-linked list across the barrier). */ +struct courier_build { + struct rb_ractor_courier *c; + st_table *seen; /* src VALUE -> (node id + 1) */ + /* Copy mode: read the sources instead of taking them apart. No husk, no buffer + * hand-over, no freeing of the source's internals. */ + bool copy; +}; + +static uint32_t courier_capture(struct courier_build *b, VALUE obj); + +/* Off the hot path: the preflight sizes both arrays, so this only runs if its count + * came out short. Swap a fresh array in rather than realloc -- the courier is a GC + * root while it is being built, and a realloc leaves the old pointer live over a + * window where it may already have been freed. */ +NOINLINE(static void courier_grow_nodes(struct rb_ractor_courier *c)); +NOINLINE(static void courier_grow_refs(struct rb_ractor_courier *c)); static void -move_courier_registry_add(struct rb_ractor_move_courier *c) +courier_grow_nodes(struct rb_ractor_courier *c) { - rb_native_mutex_lock(&GET_VM()->ractor.move_courier_registry_lock); - ccan_list_add(&GET_VM()->ractor.move_courier_registry, &c->reg_node); - rb_native_mutex_unlock(&GET_VM()->ractor.move_courier_registry_lock); + uint32_t capa = c->capa ? c->capa * 2 : 8; + struct courier_node *nodes = ALLOC_N(struct courier_node, capa); + if (c->count > 0) MEMCPY(nodes, c->nodes, struct courier_node, c->count); + struct courier_node *old_nodes = c->nodes; + c->nodes = nodes; + c->capa = capa; + ruby_xfree(old_nodes); } static void -move_courier_registry_remove(struct rb_ractor_move_courier *c) +courier_grow_refs(struct rb_ractor_courier *c) { - rb_native_mutex_lock(&GET_VM()->ractor.move_courier_registry_lock); - ccan_list_del(&c->reg_node); - rb_native_mutex_unlock(&GET_VM()->ractor.move_courier_registry_lock); + uint32_t capa = c->refs_capa ? c->refs_capa * 2 : 8; + VALUE *refs = ALLOC_N(VALUE, capa); + if (c->refs_count > 0) MEMCPY(refs, c->refs, VALUE, c->refs_count); + VALUE *old_refs = c->refs; + c->refs = refs; + c->refs_capa = capa; + ruby_xfree(old_refs); } -void rb_ractor_move_courier_mark(struct rb_ractor_move_courier *c); - -/* Called from the global GC's root pass; stop-the-world, so no lock. */ -void -rb_ractor_move_courier_registry_mark(void) +static uint32_t +courier_alloc_node(struct rb_ractor_courier *c) +{ + if (RB_UNLIKELY(c->count == c->capa)) courier_grow_nodes(c); + /* Fill the slot with a harmless REF/Qnil and bump the count only after, the way + * courier_alloc_ref does: the courier is a GC root while it is being built, and + * the mark walks nodes[0, count). A captured node overwrites this later. */ + struct courier_node *n = &c->nodes[c->count]; + n->kind = COURIER_KIND_REF; + n->frozen = false; + n->niv = 0; + n->iv_ids = NULL; + n->iv_vals = NULL; + n->u.ref = Qnil; + return c->count++; +} + +/* Size the arrays from the preflight's count, so capture never grows them. A count + * that turns out short is not a problem: the growth path below still works. */ +static void +courier_reserve(struct rb_ractor_courier *c, uint32_t nodes, uint32_t refs) { - struct rb_ractor_move_courier *c; - ccan_list_for_each(&GET_VM()->ractor.move_courier_registry, c, reg_node) { - rb_ractor_move_courier_mark(c); + if (nodes > 0) { + c->nodes = ALLOC_N(struct courier_node, nodes); + c->capa = nodes; + } + if (refs > 0) { + c->refs = ALLOC_N(VALUE, refs); + c->refs_capa = refs; } } -struct move_build { - struct rb_ractor_move_courier *c; - st_table *seen; /* src VALUE -> (node id + 1) */ -}; - -static uint32_t move_capture(struct move_build *b, VALUE obj); - +/* Embed a shareable payload by value and return its tagged child id. No dedup: a REF + * is the same word however often it appears, and an array of immediates would otherwise + * pay a lookup and an insert per element. */ static uint32_t -move_alloc_node(struct rb_ractor_move_courier *c) -{ - if (c->count == c->capa) { - c->capa = c->capa ? c->capa * 2 : 8; - REALLOC_N(c->nodes, struct move_node, c->capa); - } - uint32_t id = c->count++; - /* Initialize to a harmless REF/Qnil so the courier mark (a GC root while sending) - * is safe even mid-construction; a captured node overwrites it later. */ - c->nodes[id].kind = MOVE_KIND_REF; - c->nodes[id].frozen = false; - c->nodes[id].niv = 0; - c->nodes[id].iv_ids = NULL; - c->nodes[id].iv_vals = NULL; - c->nodes[id].u.ref = Qnil; - return id; +courier_alloc_ref(struct rb_ractor_courier *c, VALUE v) +{ + /* The count is bumped only after the slot holds a real VALUE: the courier is a GC + * root while it is being built and must never be walkable half-written. */ + if (RB_UNLIKELY(c->refs_count == c->refs_capa)) courier_grow_refs(c); + c->refs[c->refs_count] = v; + return COURIER_ID_REF_BIT | c->refs_count++; +} + +/* Resolve a child slot to the object it names. */ +static VALUE +courier_child(const struct rb_ractor_courier *c, VALUE shells, uint32_t id) +{ + if (id & COURIER_ID_REF_BIT) return c->refs[id & ~COURIER_ID_REF_BIT]; + return RARRAY_AREF(shells, id); } /* Turn a moved source into a valid RactorMovedObject without passing through flags==0, @@ -2530,25 +2567,25 @@ move_neutralize_source(VALUE obj) } } -struct move_hash_ctx { - struct move_build *b; +struct courier_hash_ctx { + struct courier_build *b; uint32_t *kv; long i; }; static int -move_capture_hash_i(st_data_t key, st_data_t val, st_data_t arg) +courier_capture_hash_i(st_data_t key, st_data_t val, st_data_t arg) { - struct move_hash_ctx *hc = (struct move_hash_ctx *)arg; - uint32_t kid = move_capture(hc->b, (VALUE)key); - uint32_t vid = move_capture(hc->b, (VALUE)val); + struct courier_hash_ctx *hc = (struct courier_hash_ctx *)arg; + uint32_t kid = courier_capture(hc->b, (VALUE)key); + uint32_t vid = courier_capture(hc->b, (VALUE)val); hc->kv[hc->i++] = kid; hc->kv[hc->i++] = vid; return ST_CONTINUE; } -struct move_obj_ctx { - struct move_build *b; +struct courier_obj_ctx { + struct courier_build *b; ID *ids; uint32_t *vals; long n; @@ -2556,15 +2593,15 @@ struct move_obj_ctx { }; static int -move_capture_ivar_i(ID name, VALUE val, st_data_t arg) +courier_capture_ivar_i(ID name, VALUE val, st_data_t arg) { - struct move_obj_ctx *oc = (struct move_obj_ctx *)arg; + struct courier_obj_ctx *oc = (struct courier_obj_ctx *)arg; if (oc->n == oc->capa) { oc->capa = oc->capa ? oc->capa * 2 : 4; REALLOC_N(oc->ids, ID, oc->capa); REALLOC_N(oc->vals, uint32_t, oc->capa); } - uint32_t vid = move_capture(oc->b, val); + uint32_t vid = courier_capture(oc->b, val); oc->ids[oc->n] = name; oc->vals[oc->n] = vid; oc->n++; @@ -2575,10 +2612,10 @@ move_capture_ivar_i(ID name, VALUE val, st_data_t arg) * Handles both a T_OBJECT's inline ivars and the generic ivars of a String, Array and * so on. */ static void -move_capture_ivars(struct move_build *b, VALUE obj, uint32_t id) +courier_capture_ivars(struct courier_build *b, VALUE obj, uint32_t id) { - struct move_obj_ctx oc = { b, NULL, NULL, 0, 0 }; - rb_ivar_foreach_buffered(obj, move_capture_ivar_i, (st_data_t)&oc); + struct courier_obj_ctx oc = { b, NULL, NULL, 0, 0 }; + rb_ivar_foreach_buffered(obj, courier_capture_ivar_i, (st_data_t)&oc); b->c->nodes[id].niv = (uint32_t)oc.n; b->c->nodes[id].iv_ids = oc.ids; b->c->nodes[id].iv_vals = oc.vals; @@ -2586,29 +2623,33 @@ move_capture_ivars(struct move_build *b, VALUE obj, uint32_t id) /* Capture obj into the courier, recurse into its children, return its node id. The id * is registered before recursing (a cycle back resolves to the same node); node fields - * are written after (recursion can realloc c->nodes); the source is neutralized exactly - * once after the switch. */ + * are written after (recursion can realloc c->nodes); a move neutralizes the source + * exactly once after the switch. */ static uint32_t -move_capture(struct move_build *b, VALUE obj) +courier_capture(struct courier_build *b, VALUE obj) { + /* An immediate is never in seen (only captured objects are inserted), so it can + * skip the lookup entirely: that is the whole cost of an array of numbers. */ + if (RB_SPECIAL_CONST_P(obj)) { + return courier_alloc_ref(b->c, obj); + } + + /* Seen first, and only then shareable: move husks each source as it goes, and a + * husk is a frozen field-less object, which rb_ractor_shareable_p answers true for. + * Testing shareable first would embed the husk instead of resolving the second + * occurrence to the node the first one built. */ st_data_t existing; if (st_lookup(b->seen, (st_data_t)obj, &existing)) { return (uint32_t)existing - 1; } - uint32_t id = move_alloc_node(b->c); - st_insert(b->seen, (st_data_t)obj, (st_data_t)(uintptr_t)(id + 1)); - - if (RB_SPECIAL_CONST_P(obj) || rb_ractor_shareable_p(obj)) { - b->c->nodes[id].kind = MOVE_KIND_REF; - b->c->nodes[id].frozen = false; - b->c->nodes[id].niv = 0; - b->c->nodes[id].iv_ids = NULL; - b->c->nodes[id].iv_vals = NULL; - b->c->nodes[id].u.ref = obj; - return id; + if (rb_ractor_shareable_p(obj)) { + return courier_alloc_ref(b->c, obj); } + uint32_t id = courier_alloc_node(b->c); + st_insert(b->seen, (st_data_t)obj, (st_data_t)(uintptr_t)(id + 1)); + /* Reject an unmovable object before anything is mutated. */ if (BUILTIN_TYPE(obj) == T_FILE && RFILE(obj)->fptr == NULL) { rb_raise(rb_eRactorError, "can not move an uninitialized IO"); @@ -2616,34 +2657,41 @@ move_capture(struct move_build *b, VALUE obj) bool frozen = OBJ_FROZEN(obj); b->c->nodes[id].frozen = frozen; - move_capture_ivars(b, obj, id); /* shared: instance and generic ivars */ + courier_capture_ivars(b, obj, id); /* shared: instance and generic ivars */ switch (BUILTIN_TYPE(obj)) { case T_STRING: { /* Give the source its own buffer (drop sharing, copy a static STR_NOFREE one). * Safe even when frozen: it changes ownership, not content. Afterwards a string * is embedded, owns a private heap buffer, or is a shared ROOT (a no-op). */ - rb_str_make_independent(obj); + if (!b->copy) rb_str_make_independent(obj); long len = RSTRING_LEN(obj); int encidx = ENCODING_GET(obj); + /* The receiver adopts this buffer as a String body, which is freed by size: + * capa has to describe the allocation exactly (capa + terminator bytes). */ + const int termlen = rb_enc_mbminlen(rb_enc_from_index(encidx)); char *ptr; - if (!STR_EMBED_P(obj) && rb_str_reembeddable_p(obj)) { + long capa; + if (!b->copy && !STR_EMBED_P(obj) && rb_str_reembeddable_p(obj)) { /* Owns a private heap buffer: carry the pointer over (zero-copy) and leave * the source as a shell that does not free it. */ ptr = RSTRING(obj)->as.heap.ptr; + capa = RSTRING(obj)->as.heap.aux.capa; } else { /* Embedded or a shared root: copy the bytes into a courier-owned buffer. * Taking a root's buffer would dangle its copy-on-write children, so leave * it (the same reason T_ARRAY excludes ARY_SHARED_ROOT_P below). */ - ptr = ALLOC_N(char, len + 1); + ptr = ALLOC_N(char, len + termlen); if (len) memcpy(ptr, RSTRING_PTR(obj), len); - ptr[len] = '\0'; + memset(ptr + len, 0, termlen); + capa = len; } - b->c->nodes[id].kind = MOVE_KIND_STRING; + b->c->nodes[id].kind = COURIER_KIND_STRING; b->c->nodes[id].u.str.klass = RBASIC_CLASS(obj); b->c->nodes[id].u.str.ptr = ptr; b->c->nodes[id].u.str.len = len; + b->c->nodes[id].u.str.capa = capa; b->c->nodes[id].u.str.encidx = encidx; break; } @@ -2652,28 +2700,28 @@ move_capture(struct move_build *b, VALUE obj) long len = RARRAY_LEN(obj); uint32_t *elems = len ? ALLOC_N(uint32_t, len) : NULL; for (long i = 0; i < len; i++) { - elems[i] = move_capture(b, RARRAY_AREF(obj, i)); + elems[i] = courier_capture(b, RARRAY_AREF(obj, i)); } - b->c->nodes[id].kind = MOVE_KIND_ARRAY; + b->c->nodes[id].kind = COURIER_KIND_ARRAY; b->c->nodes[id].u.ary.klass = RBASIC_CLASS(obj); b->c->nodes[id].u.ary.len = len; b->c->nodes[id].u.ary.elems = elems; /* Free the source's heap buffer now that the children were read, but only when it * is private: a sharer's belongs to its root, a root's to its sharers -- and a * frozen array is a root without carrying the flag. */ - if (!ARY_EMBED_P(obj) && !ARY_SHARED_P(obj) && !ARY_SHARED_ROOT_P(obj) && !OBJ_FROZEN(obj)) { + if (!b->copy && !ARY_EMBED_P(obj) && !ARY_SHARED_P(obj) && !ARY_SHARED_ROOT_P(obj) && !OBJ_FROZEN(obj)) { ruby_xfree((void *)RARRAY_CONST_PTR(obj)); } break; } case T_HASH: { - uint32_t ifnone_id = move_capture(b, RHASH_IFNONE(obj)); + uint32_t ifnone_id = courier_capture(b, RHASH_IFNONE(obj)); long size = RHASH_SIZE(obj); uint32_t *kv = size ? ALLOC_N(uint32_t, size * 2) : NULL; - struct move_hash_ctx hc = { b, kv, 0 }; - rb_hash_stlike_foreach(obj, move_capture_hash_i, (st_data_t)&hc); - b->c->nodes[id].kind = MOVE_KIND_HASH; + struct courier_hash_ctx hc = { b, kv, 0 }; + rb_hash_stlike_foreach(obj, courier_capture_hash_i, (st_data_t)&hc); + b->c->nodes[id].kind = COURIER_KIND_HASH; b->c->nodes[id].u.hash.klass = RBASIC_CLASS(obj); b->c->nodes[id].u.hash.size = size; b->c->nodes[id].u.hash.kv = kv; @@ -2681,12 +2729,12 @@ move_capture(struct move_build *b, VALUE obj) b->c->nodes[id].u.hash.compare_by_id = RTEST(rb_hash_compare_by_id_p(obj)); b->c->nodes[id].u.hash.proc_default = FL_TEST_RAW(obj, RHASH_PROC_DEFAULT) != 0; /* Free the source's st-table internals (an ar table lives in the slot) */ - rb_hash_free(obj); + if (!b->copy) rb_hash_free(obj); break; } case T_OBJECT: - b->c->nodes[id].kind = MOVE_KIND_OBJECT; + b->c->nodes[id].kind = COURIER_KIND_OBJECT; /* Keep the real class: even a singleton class is shareable, so a cross-objspace * reference is safe. rebuild re-attaches it after allocating with a * non-singleton class. */ @@ -2697,14 +2745,14 @@ move_capture(struct move_build *b, VALUE obj) long len = RSTRUCT_LEN(obj); uint32_t *elems = len ? ALLOC_N(uint32_t, len) : NULL; for (long i = 0; i < len; i++) { - elems[i] = move_capture(b, RSTRUCT_GET(obj, (int)i)); + elems[i] = courier_capture(b, RSTRUCT_GET(obj, (int)i)); } - b->c->nodes[id].kind = MOVE_KIND_STRUCT; + b->c->nodes[id].kind = COURIER_KIND_STRUCT; b->c->nodes[id].u.strct.len = len; b->c->nodes[id].u.strct.elems = elems; b->c->nodes[id].u.strct.klass = RBASIC_CLASS(obj); /* Free the source's private heap buffer (an embedded struct has none) */ - if (RSTRUCT_EMBED_LEN(obj) == 0) { + if (!b->copy && RSTRUCT_EMBED_LEN(obj) == 0) { ruby_xfree((void *)RSTRUCT_CONST_PTR(obj)); } break; @@ -2715,10 +2763,10 @@ move_capture(struct move_build *b, VALUE obj) * registers (freeing the source's onig and char_offset). */ VALUE re, st; int nregs; - void *regs = rb_match_move_dump(obj, &re, &st, &nregs); - uint32_t rid = move_capture(b, re); - uint32_t sid = move_capture(b, st); - b->c->nodes[id].kind = MOVE_KIND_MATCH; + void *regs = rb_match_blob_dump(obj, &re, &st, &nregs, !b->copy); + uint32_t rid = courier_capture(b, re); + uint32_t sid = courier_capture(b, st); + b->c->nodes[id].kind = COURIER_KIND_MATCH; b->c->nodes[id].u.match.regexp_id = rid; b->c->nodes[id].u.match.str_id = sid; b->c->nodes[id].u.match.num_regs = nregs; @@ -2729,16 +2777,17 @@ move_capture(struct move_build *b, VALUE obj) case T_FILE: { + VM_ASSERT(!b->copy); /* copy_courier_supported_p rejects it */ /* Carry the whole fptr (fd included) by pointer; the source shell does not * close it. fptr's VALUE members lose their root once the source is T_MOVED, * so capture them as ordinary child nodes, detached; rebuild writes them back. */ struct rb_io *fptr = RFILE(obj)->fptr; VM_ASSERT(!RTEST(fptr->tied_io_for_writing) && !RTEST(fptr->wakeup_mutex)); - uint32_t pathv_id = move_capture(b, fptr->pathv); - uint32_t ecopts_id = move_capture(b, fptr->encs.ecopts); - uint32_t wc_pre_id = move_capture(b, fptr->writeconv_pre_ecopts); - uint32_t wc_ac_id = move_capture(b, fptr->writeconv_asciicompat); - uint32_t timeout_id = move_capture(b, fptr->timeout); + uint32_t pathv_id = courier_capture(b, fptr->pathv); + uint32_t ecopts_id = courier_capture(b, fptr->encs.ecopts); + uint32_t wc_pre_id = courier_capture(b, fptr->writeconv_pre_ecopts); + uint32_t wc_ac_id = courier_capture(b, fptr->writeconv_asciicompat); + uint32_t timeout_id = courier_capture(b, fptr->timeout); fptr->self = Qnil; /* it points at the moved-from T_MOVED; attach rebuilds it */ fptr->pathv = Qnil; fptr->encs.ecopts = Qnil; @@ -2748,7 +2797,7 @@ move_capture(struct move_build *b, VALUE obj) fptr->write_lock = Qnil; fptr->wakeup_mutex = Qnil; fptr->tied_io_for_writing = 0; /* io.c tests it as a C boolean, so 0 rather than Qnil */ - b->c->nodes[id].kind = MOVE_KIND_IO; + b->c->nodes[id].kind = COURIER_KIND_IO; b->c->nodes[id].u.io.fptr = fptr; b->c->nodes[id].u.io.klass = RBASIC_CLASS(obj); b->c->nodes[id].u.io.pathv_id = pathv_id; @@ -2759,41 +2808,65 @@ move_capture(struct move_build *b, VALUE obj) break; } + case T_DATA: + /* Only an exception's backtrace, and only for a copy: move still refuses every + * T_DATA (its source would have to be taken apart). */ + if (b->copy && rb_backtrace_p(obj)) { + int size; + void *blob = rb_backtrace_blob_dump(obj, &size); + b->c->nodes[id].kind = COURIER_KIND_BACKTRACE; + b->c->nodes[id].u.bt.blob = blob; + b->c->nodes[id].u.bt.size = size; + break; + } + /* fall through */ default: rb_raise(rb_eRactorError, "can not move a %"PRIsVALUE" object", rb_class_name(rb_obj_class(obj))); } - move_neutralize_source(obj); + if (!b->copy) move_neutralize_source(obj); return id; } -static void move_preflight(VALUE obj, st_table *seen); +/* Like the copy walk, this also sizes the courier: see copy_support_ctx. */ +struct move_preflight_ctx { + st_table *seen; + uint32_t nodes, refs; +}; + +static void move_preflight(VALUE obj, struct move_preflight_ctx *ctx); static int move_preflight_ivar_i(ID name, VALUE val, st_data_t arg) { - move_preflight(val, (st_table *)arg); + move_preflight(val, (struct move_preflight_ctx *)arg); return ST_CONTINUE; } static int move_preflight_hash_i(st_data_t key, st_data_t val, st_data_t arg) { - move_preflight((VALUE)key, (st_table *)arg); - move_preflight((VALUE)val, (st_table *)arg); + move_preflight((VALUE)key, (struct move_preflight_ctx *)arg); + move_preflight((VALUE)val, (struct move_preflight_ctx *)arg); return ST_CONTINUE; } -/* A read-only pre-walk of move_capture's decision tree. Capture turns sources into +/* A read-only pre-walk of courier_capture's decision tree. Capture turns sources into * T_MOVED as it goes, so an unmovable object midway would leave the graph broken beyond * repair; every "can not move" error is raised here, before anything is mutated. */ static void -move_preflight(VALUE obj, st_table *seen) +move_preflight(VALUE obj, struct move_preflight_ctx *ctx) { - if (RB_SPECIAL_CONST_P(obj) || rb_ractor_shareable_p(obj)) return; + st_table *const seen = ctx->seen; + + if (RB_SPECIAL_CONST_P(obj) || rb_ractor_shareable_p(obj)) { + ctx->refs++; + return; + } if (st_lookup(seen, (st_data_t)obj, NULL)) return; /* cycle */ st_insert(seen, (st_data_t)obj, 0); + ctx->nodes++; switch (BUILTIN_TYPE(obj)) { case T_STRING: @@ -2801,22 +2874,22 @@ move_preflight(VALUE obj, st_table *seen) break; /* children are ivars only (below) */ case T_MATCH: { struct RMatch *rm = RMATCH(obj); - move_preflight(rm->regexp, seen); - move_preflight(rm->str, seen); + move_preflight(rm->regexp, ctx); + move_preflight(rm->str, ctx); break; } case T_ARRAY: for (long i = 0; i < RARRAY_LEN(obj); i++) { - move_preflight(RARRAY_AREF(obj, i), seen); + move_preflight(RARRAY_AREF(obj, i), ctx); } break; case T_HASH: - rb_hash_stlike_foreach(obj, move_preflight_hash_i, (st_data_t)seen); - move_preflight(RHASH_IFNONE(obj), seen); + rb_hash_stlike_foreach(obj, move_preflight_hash_i, (st_data_t)ctx); + move_preflight(RHASH_IFNONE(obj), ctx); break; case T_STRUCT: for (long i = 0; i < RSTRUCT_LEN(obj); i++) { - move_preflight(RSTRUCT_GET(obj, (int)i), seen); + move_preflight(RSTRUCT_GET(obj, (int)i), ctx); } break; case T_FILE: { @@ -2833,11 +2906,11 @@ move_preflight(VALUE obj, st_table *seen) /* A close is in progress: a thread is blocked on this IO. */ rb_raise(rb_eRactorError, "can not move an IO that is being closed"); } - move_preflight(fptr->pathv, seen); - move_preflight(fptr->encs.ecopts, seen); - move_preflight(fptr->writeconv_pre_ecopts, seen); - move_preflight(fptr->writeconv_asciicompat, seen); - move_preflight(fptr->timeout, seen); + move_preflight(fptr->pathv, ctx); + move_preflight(fptr->encs.ecopts, ctx); + move_preflight(fptr->writeconv_pre_ecopts, ctx); + move_preflight(fptr->writeconv_asciicompat, ctx); + move_preflight(fptr->timeout, ctx); break; } default: @@ -2845,51 +2918,176 @@ move_preflight(VALUE obj, st_table *seen) rb_class_name(rb_obj_class(obj))); } - rb_ivar_foreach(obj, move_preflight_ivar_i, (st_data_t)seen); + rb_ivar_foreach(obj, move_preflight_ivar_i, (st_data_t)ctx); +} + +/* The walk also sizes the courier: one node per distinct unshareable object, one ref + * per occurrence of a shareable one -- exactly what courier_capture allocates, so the + * arrays never have to grow while the graph is being captured. */ +struct copy_support_ctx { + st_table *seen; + uint32_t nodes, refs; + bool ok; +}; + +static bool copy_courier_supported_p(VALUE obj, struct copy_support_ctx *ctx); + +static int +copy_support_val_i(st_data_t val, st_data_t arg) +{ + struct copy_support_ctx *ctx = (struct copy_support_ctx *)arg; + if (!copy_courier_supported_p((VALUE)val, ctx)) { + ctx->ok = false; + return ST_STOP; + } + return ST_CONTINUE; +} + +static int +copy_support_ivar_i(ID name, VALUE val, st_data_t arg) +{ + return copy_support_val_i((st_data_t)val, arg); +} + +static int +copy_support_hash_i(st_data_t key, st_data_t val, st_data_t arg) +{ + if (copy_support_val_i(key, arg) == ST_STOP) return ST_STOP; + return copy_support_val_i(val, arg); +} + +/* Read-only walk: can the copy courier carry obj's whole graph? Everything it says no + * to (MatchData, IO, any other T_DATA, a singleton class) stays on the older on-heap + * snapshot path, which keeps handling or rejecting it exactly as before. */ +static bool +copy_courier_supported_p(VALUE obj, struct copy_support_ctx *ctx) +{ + st_table *const seen = ctx->seen; + + if (RB_SPECIAL_CONST_P(obj) || rb_ractor_shareable_p(obj)) { + ctx->refs++; + return true; + } + if (st_lookup(seen, (st_data_t)obj, NULL)) return true; /* cycle */ + st_insert(seen, (st_data_t)obj, 0); + ctx->nodes++; + + /* A singleton class is a send error today (the native copier refuses it and Marshal + * then raises); the courier would happily carry it, so keep it off this path. */ + VALUE klass = RBASIC_CLASS(obj); + if (klass == 0 || FL_TEST_RAW(klass, FL_SINGLETON)) return false; + + switch (BUILTIN_TYPE(obj)) { + case T_STRING: + case T_OBJECT: + break; /* children are ivars only (below) */ + case T_MATCH: { + struct RMatch *rm = RMATCH(obj); + if (!copy_courier_supported_p(rm->regexp, ctx)) return false; + if (!copy_courier_supported_p(rm->str, ctx)) return false; + break; + } + case T_DATA: + /* An exception's backtrace is the one T_DATA the courier carries. */ + if (!rb_backtrace_p(obj)) return false; + break; + case T_ARRAY: + for (long i = 0; i < RARRAY_LEN(obj); i++) { + if (!copy_courier_supported_p(RARRAY_AREF(obj, i), ctx)) return false; + } + break; + case T_HASH: + rb_hash_stlike_foreach(obj, copy_support_hash_i, (st_data_t)ctx); + if (!ctx->ok) return false; + if (!copy_courier_supported_p(RHASH_IFNONE(obj), ctx)) return false; + break; + case T_STRUCT: + for (long i = 0; i < RSTRUCT_LEN(obj); i++) { + if (!copy_courier_supported_p(RSTRUCT_GET(obj, (int)i), ctx)) return false; + } + break; + default: + return false; + } + + rb_ivar_foreach(obj, copy_support_ivar_i, (st_data_t)ctx); + return ctx->ok; } -/* Build a move courier from obj and turn every captured source into a - * RactorMovedObject (move semantics). Returns the xmalloc'd courier. */ -struct rb_ractor_move_courier * -rb_ractor_move_courier_build(VALUE obj) +/* Build a courier holding a copy of obj's graph, leaving the sources untouched. + * Returns NULL when the graph has a type only the on-heap snapshot path handles. */ +struct rb_ractor_courier * +rb_ractor_courier_build_copy(VALUE obj, struct rb_ractor_courier **slot) +{ + struct copy_support_ctx scan = { st_init_numtable(), 0, 0, true }; + { + bool ok = copy_courier_supported_p(obj, &scan); + st_free_table(scan.seen); + if (!ok) return NULL; + } + + struct rb_ractor_courier *c = ZALLOC(struct rb_ractor_courier); + courier_reserve(c, scan.nodes, scan.refs); + struct courier_build b = { c, st_init_numtable(), true }; + + /* Publish it into the caller's basket before capturing anything: from here the + * shareable payloads it collects are rooted by the basket's holder. */ + *slot = c; + + enum ruby_tag_type state; + rb_execution_context_t *ec = GET_EC(); + EC_PUSH_TAG(ec); + if ((state = EC_EXEC_TAG()) == TAG_NONE) { + c->root = courier_capture(&b, obj); + } + EC_POP_TAG(); + st_free_table(b.seen); + /* Published above, so the basket owns it even half-built: it frees it. */ + if (state != TAG_NONE) EC_JUMP_TAG(ec, state); + return c; +} + +/* Build a courier from obj and turn every captured source into a RactorMovedObject + * (move semantics). Returns the xmalloc'd courier. */ +struct rb_ractor_courier * +rb_ractor_courier_build_move(VALUE obj, struct rb_ractor_courier **slot) { /* Two phases, preflight then commit, so an unmovable object is raised from the * read-only walk while the graph is still intact. */ + struct move_preflight_ctx scan = { st_init_numtable(), 0, 0 }; { - st_table *pf_seen = st_init_numtable(); enum ruby_tag_type state; rb_execution_context_t *ec = GET_EC(); EC_PUSH_TAG(ec); if ((state = EC_EXEC_TAG()) == TAG_NONE) { - move_preflight(obj, pf_seen); + move_preflight(obj, &scan); } EC_POP_TAG(); - st_free_table(pf_seen); + st_free_table(scan.seen); if (state != TAG_NONE) EC_JUMP_TAG(ec, state); } - struct rb_ractor_move_courier *c = ZALLOC(struct rb_ractor_move_courier); - struct move_build b = { c, st_init_numtable() }; + struct rb_ractor_courier *c = ZALLOC(struct rb_ractor_courier); + courier_reserve(c, scan.nodes, scan.refs); + struct courier_build b = { c, st_init_numtable(), false }; - /* Between send and materialization the courier's shareable REFs pass through - * windows where nothing else roots them; register it for its whole lifetime so the - * registry root pass marks and pins them. Registering before the sources become - * T_MOVED is safe: partial nodes are initialized mark-safe. */ - move_courier_registry_add(c); + /* Publish it into the caller's basket before the sources become T_MOVED: from here + * the basket's holder roots what the courier carries, and partial nodes are + * initialized mark-safe. */ + *slot = c; enum ruby_tag_type state; rb_execution_context_t *ec = GET_EC(); EC_PUSH_TAG(ec); if ((state = EC_EXEC_TAG()) == TAG_NONE) { - c->root = move_capture(&b, obj); + c->root = courier_capture(&b, obj); } EC_POP_TAG(); st_free_table(b.seen); if (state != TAG_NONE) { - /* move_capture raised (an unmovable type, an interrupt). Remove the courier - * from the registry and free it before re-raising; the partial nodes are - * mark-safe and safe to free. */ - rb_ractor_move_courier_free(c); + /* courier_capture raised (an unmovable type, an interrupt). The courier belongs + * to the basket from the publish above, so leave it there and re-raise: the + * basket frees it, once, on the way out. */ EC_JUMP_TAG(ec, state); } return c; @@ -2899,7 +3097,7 @@ rb_ractor_move_courier_build(VALUE obj) * singleton class (classes are shareable; the reference is safe). A singleton's * attached object still points at the sender's source: re-attach it to the shell. */ static void -move_apply_moved_klass(VALUE shell, VALUE klass) +courier_apply_klass(VALUE shell, VALUE klass) { if (klass != RBASIC_CLASS(shell)) { RBASIC_SET_CLASS(shell, klass); @@ -2912,91 +3110,105 @@ move_apply_moved_klass(VALUE shell, VALUE klass) /* Rebuild the courier's graph in the current Ractor's objspace and return its root. * Two passes (allocate shells, then fill) break reference cycles. */ VALUE -rb_ractor_move_courier_materialize(struct rb_ractor_move_courier *c) +rb_ractor_courier_materialize(struct rb_ractor_courier *c) { /* A hidden Array roots every shell, keeping them alive while the allocations that * build the rest of the graph (which can start this Ractor's GC) run. */ VALUE shells = rb_ary_hidden_new(c->count); for (uint32_t i = 0; i < c->count; i++) { - struct move_node *n = &c->nodes[i]; + struct courier_node *n = &c->nodes[i]; VALUE shell; switch (n->kind) { - case MOVE_KIND_REF: + case COURIER_KIND_REF: shell = n->u.ref; break; - case MOVE_KIND_STRING: - shell = rb_enc_str_new(n->u.str.ptr, n->u.str.len, rb_enc_from_index(n->u.str.encidx)); - move_apply_moved_klass(shell, n->u.str.klass); + case COURIER_KIND_STRING: + /* Hand the courier's buffer to the String instead of copying it again: the + * bytes were already copied (or taken from the source) when the node was + * built. */ + shell = rb_str_new_owned(n->u.str.ptr, n->u.str.len, n->u.str.capa, n->u.str.encidx); + n->u.str.ptr = NULL; /* consumed: the new String owns it now */ + courier_apply_klass(shell, n->u.str.klass); break; - case MOVE_KIND_ARRAY: + case COURIER_KIND_ARRAY: shell = rb_ary_new_capa(n->u.ary.len); - move_apply_moved_klass(shell, n->u.ary.klass); + courier_apply_klass(shell, n->u.ary.klass); break; - case MOVE_KIND_HASH: + case COURIER_KIND_HASH: shell = n->u.hash.compare_by_id ? rb_ident_hash_new() : rb_hash_new(); - move_apply_moved_klass(shell, n->u.hash.klass); + courier_apply_klass(shell, n->u.hash.klass); break; - case MOVE_KIND_OBJECT: + case COURIER_KIND_OBJECT: /* A singleton class cannot allocate, so make an instance of the real class * and re-attach it afterwards */ shell = rb_obj_alloc(rb_class_real(n->u.obj.klass)); - move_apply_moved_klass(shell, n->u.obj.klass); + courier_apply_klass(shell, n->u.obj.klass); break; - case MOVE_KIND_STRUCT: + case COURIER_KIND_STRUCT: shell = rb_obj_alloc(rb_class_real(n->u.strct.klass)); - move_apply_moved_klass(shell, n->u.strct.klass); + courier_apply_klass(shell, n->u.strct.klass); break; - case MOVE_KIND_MATCH: - shell = rb_match_move_alloc(rb_class_real(n->u.match.klass), n->u.match.num_regs); - move_apply_moved_klass(shell, n->u.match.klass); + case COURIER_KIND_MATCH: + shell = rb_match_blob_alloc(rb_class_real(n->u.match.klass), n->u.match.num_regs); + courier_apply_klass(shell, n->u.match.klass); break; - case MOVE_KIND_IO: + case COURIER_KIND_BACKTRACE: + shell = rb_backtrace_blob_load(n->u.bt.blob, n->u.bt.size); + break; + case COURIER_KIND_IO: shell = rb_obj_alloc(rb_class_real(n->u.io.klass)); - move_apply_moved_klass(shell, n->u.io.klass); + courier_apply_klass(shell, n->u.io.klass); RFILE(shell)->fptr = n->u.io.fptr; n->u.io.fptr->self = shell; n->u.io.fptr = NULL; /* consumed: the new IO owns it now */ break; default: - rb_bug("rb_ractor_move_courier_materialize: bad node kind"); + rb_bug("rb_ractor_courier_materialize: bad node kind"); } rb_ary_push(shells, shell); } for (uint32_t i = 0; i < c->count; i++) { - struct move_node *n = &c->nodes[i]; + struct courier_node *n = &c->nodes[i]; VALUE shell = RARRAY_AREF(shells, i); switch (n->kind) { - case MOVE_KIND_ARRAY: - for (long j = 0; j < n->u.ary.len; j++) { - rb_ary_push(shell, RARRAY_AREF(shells, n->u.ary.elems[j])); + case COURIER_KIND_ARRAY: { + /* The length is known, so set it once and write the slots, rather than + * pushing each element through the capacity check. */ + const long len = n->u.ary.len; + if (len > 0) { + rb_ary_resize(shell, len); + for (long j = 0; j < len; j++) { + RARRAY_ASET(shell, j, courier_child(c, shells, n->u.ary.elems[j])); + } } break; - case MOVE_KIND_HASH: + } + case COURIER_KIND_HASH: /* Entry insertion is deferred to a third pass: insertion calls the key's * #hash / #eql?, and a content-based #hash would collide on every key while * the graph is still empty, collapsing entries. */ break; - case MOVE_KIND_STRUCT: + case COURIER_KIND_STRUCT: for (long j = 0; j < n->u.strct.len; j++) { - RSTRUCT_SET(shell, (int)j, RARRAY_AREF(shells, n->u.strct.elems[j])); + RSTRUCT_SET(shell, (int)j, courier_child(c, shells, n->u.strct.elems[j])); } break; - case MOVE_KIND_MATCH: - rb_match_move_load(shell, RARRAY_AREF(shells, n->u.match.regexp_id), - RARRAY_AREF(shells, n->u.match.str_id), + case COURIER_KIND_MATCH: + rb_match_blob_load(shell, courier_child(c, shells, n->u.match.regexp_id), + courier_child(c, shells, n->u.match.str_id), n->u.match.num_regs, n->u.match.regs); break; - case MOVE_KIND_IO: { + case COURIER_KIND_IO: { /* Write the rebuilt VALUE members back into fptr (capture detached them). * write_lock and wakeup_mutex stay nil; io.c recreates them lazily. */ struct rb_io *fptr = RFILE(shell)->fptr; - RB_OBJ_WRITE(shell, &fptr->pathv, RARRAY_AREF(shells, n->u.io.pathv_id)); - RB_OBJ_WRITE(shell, &fptr->encs.ecopts, RARRAY_AREF(shells, n->u.io.ecopts_id)); - RB_OBJ_WRITE(shell, &fptr->writeconv_pre_ecopts, RARRAY_AREF(shells, n->u.io.wc_pre_ecopts_id)); - RB_OBJ_WRITE(shell, &fptr->writeconv_asciicompat, RARRAY_AREF(shells, n->u.io.wc_asciicompat_id)); - RB_OBJ_WRITE(shell, &fptr->timeout, RARRAY_AREF(shells, n->u.io.timeout_id)); + RB_OBJ_WRITE(shell, &fptr->pathv, courier_child(c, shells, n->u.io.pathv_id)); + RB_OBJ_WRITE(shell, &fptr->encs.ecopts, courier_child(c, shells, n->u.io.ecopts_id)); + RB_OBJ_WRITE(shell, &fptr->writeconv_pre_ecopts, courier_child(c, shells, n->u.io.wc_pre_ecopts_id)); + RB_OBJ_WRITE(shell, &fptr->writeconv_asciicompat, courier_child(c, shells, n->u.io.wc_asciicompat_id)); + RB_OBJ_WRITE(shell, &fptr->timeout, courier_child(c, shells, n->u.io.timeout_id)); break; } default: @@ -3004,7 +3216,7 @@ rb_ractor_move_courier_materialize(struct rb_ractor_move_courier *c) } /* Restore instance and generic ivars (any non-REF node can have them) */ for (uint32_t j = 0; j < n->niv; j++) { - rb_ivar_set(shell, n->iv_ids[j], RARRAY_AREF(shells, n->iv_vals[j])); + rb_ivar_set(shell, n->iv_ids[j], courier_child(c, shells, n->iv_vals[j])); } } @@ -3012,15 +3224,15 @@ rb_ractor_move_courier_materialize(struct rb_ractor_move_courier *c) * depth-first (children larger), so inserting in reverse settles nested hash keys * inside-out (a #hash cycling through itself is out of scope). */ for (uint32_t i = c->count; i > 0; i--) { - struct move_node *n = &c->nodes[i - 1]; - if (n->kind != MOVE_KIND_HASH) continue; + struct courier_node *n = &c->nodes[i - 1]; + if (n->kind != COURIER_KIND_HASH) continue; VALUE shell = RARRAY_AREF(shells, i - 1); for (long j = 0; j < n->u.hash.size; j++) { - rb_hash_aset(shell, RARRAY_AREF(shells, n->u.hash.kv[2 * j]), - RARRAY_AREF(shells, n->u.hash.kv[2 * j + 1])); + rb_hash_aset(shell, courier_child(c, shells, n->u.hash.kv[2 * j]), + courier_child(c, shells, n->u.hash.kv[2 * j + 1])); } /* Restore the default value and default proc (before freezing) */ - VALUE ifnone = RARRAY_AREF(shells, n->u.hash.ifnone_id); + VALUE ifnone = courier_child(c, shells, n->u.hash.ifnone_id); if (n->u.hash.proc_default) { rb_hash_set_default_proc(shell, ifnone); } @@ -3037,35 +3249,38 @@ rb_ractor_move_courier_materialize(struct rb_ractor_move_courier *c) } } - VALUE root = c->count ? RARRAY_AREF(shells, c->root) : Qnil; + VALUE root = (c->count || c->refs_count) ? courier_child(c, shells, c->root) : Qnil; RB_GC_GUARD(shells); return root; } void -rb_ractor_move_courier_free(struct rb_ractor_move_courier *c) +rb_ractor_courier_free(struct rb_ractor_courier *c) { for (uint32_t i = 0; i < c->count; i++) { - struct move_node *n = &c->nodes[i]; + struct courier_node *n = &c->nodes[i]; ruby_xfree(n->iv_ids); ruby_xfree(n->iv_vals); switch (n->kind) { - case MOVE_KIND_STRING: + case COURIER_KIND_STRING: ruby_xfree(n->u.str.ptr); break; - case MOVE_KIND_ARRAY: + case COURIER_KIND_ARRAY: ruby_xfree(n->u.ary.elems); break; - case MOVE_KIND_HASH: + case COURIER_KIND_HASH: ruby_xfree(n->u.hash.kv); break; - case MOVE_KIND_STRUCT: + case COURIER_KIND_STRUCT: ruby_xfree(n->u.strct.elems); break; - case MOVE_KIND_MATCH: - rb_match_move_free(n->u.match.regs); + case COURIER_KIND_MATCH: + rb_match_blob_free(n->u.match.regs); + break; + case COURIER_KIND_BACKTRACE: + ruby_xfree(n->u.bt.blob); break; - case MOVE_KIND_IO: + case COURIER_KIND_IO: /* A delivered IO left fptr == NULL (the rebuilt IO owns it). An * undelivered one still owns the fd and its source is already a * RactorMovedObject nobody can close: close it here, not leak it. */ @@ -3078,8 +3293,8 @@ rb_ractor_move_courier_free(struct rb_ractor_move_courier *c) break; } } - move_courier_registry_remove(c); ruby_xfree(c->nodes); + ruby_xfree(c->refs); ruby_xfree(c); } @@ -3087,33 +3302,39 @@ rb_ractor_move_courier_free(struct rb_ractor_move_courier *c) * classes of its objects. All of them are shareable, so marking cannot race, and the * global GC keeps them reachable through the courier. */ void -rb_ractor_move_courier_mark(struct rb_ractor_move_courier *c) +rb_ractor_courier_mark(struct rb_ractor_courier *c) { if (!c) return; + for (uint32_t i = 0; i < c->refs_count; i++) { + rb_gc_mark(c->refs[i]); + } for (uint32_t i = 0; i < c->count; i++) { - struct move_node *n = &c->nodes[i]; - if (n->kind == MOVE_KIND_REF) { + struct courier_node *n = &c->nodes[i]; + if (n->kind == COURIER_KIND_REF) { rb_gc_mark(n->u.ref); } - else if (n->kind == MOVE_KIND_OBJECT) { + else if (n->kind == COURIER_KIND_OBJECT) { rb_gc_mark(n->u.obj.klass); } - else if (n->kind == MOVE_KIND_STRUCT) { + else if (n->kind == COURIER_KIND_STRUCT) { rb_gc_mark(n->u.strct.klass); } - else if (n->kind == MOVE_KIND_MATCH) { + else if (n->kind == COURIER_KIND_MATCH) { rb_gc_mark(n->u.match.klass); } - else if (n->kind == MOVE_KIND_IO) { + else if (n->kind == COURIER_KIND_IO) { rb_gc_mark(n->u.io.klass); } - else if (n->kind == MOVE_KIND_STRING) { + else if (n->kind == COURIER_KIND_STRING) { rb_gc_mark(n->u.str.klass); } - else if (n->kind == MOVE_KIND_ARRAY) { + else if (n->kind == COURIER_KIND_BACKTRACE) { + rb_backtrace_blob_mark(n->u.bt.blob, n->u.bt.size); + } + else if (n->kind == COURIER_KIND_ARRAY) { rb_gc_mark(n->u.ary.klass); } - else if (n->kind == MOVE_KIND_HASH) { + else if (n->kind == COURIER_KIND_HASH) { rb_gc_mark(n->u.hash.klass); } } @@ -3184,21 +3405,6 @@ ractor_native_shallow_copy(VALUE obj) return copy; } -/* Add a node of the snapshot under construction to the pin list and pin it now. */ -static void -ractor_pin_capture_push(rb_ractor_t *cr, VALUE v) -{ - if (cr->pin_capture_cnt == cr->pin_capture_capa) { - size_t nc = cr->pin_capture_capa ? cr->pin_capture_capa * 2 : 16; - VALUE *p = realloc(cr->pin_capture, nc * sizeof(VALUE)); - if (!p) rb_bug("ractor_pin_capture_push: out of memory"); - cr->pin_capture = p; - cr->pin_capture_capa = nc; - } - cr->pin_capture[cr->pin_capture_cnt++] = v; - rb_gc_pin_in_flight_message(v); -} - static enum obj_traverse_iterator_result copy_enter(VALUE obj, struct obj_traverse_replace_data *data) { @@ -3210,17 +3416,6 @@ copy_enter(VALUE obj, struct obj_traverse_replace_data *data) VALUE copy = ractor_native_shallow_copy(obj); if (UNDEF_P(copy)) return traverse_stop; /* no native copy for this type */ data->replacement = copy; - /* Collect every node into the pin list as the snapshot is built: the global - * GC's re-pin must cover all nodes, not just the root (moving one breaks the - * address-keyed dedup table). fields_obj is not included: the global - * generic_fields table reaches it and compaction updates that. */ - rb_ractor_t *cr = GET_RACTOR(); - if (cr->gen_fields_capturing) { - /* Pin from birth (shref bit, plus the pin bit during a global compaction). - * rb_ractor_repin_in_flight re-pins via cr->pin_capture, so the cover runs - * unbroken from construction through enqueue to materialization. */ - ractor_pin_capture_push(cr, copy); - } return traverse_cont; } } diff --git a/ractor_core.h b/ractor_core.h index 6ab440a680969c..21787329ce817d 100644 --- a/ractor_core.h +++ b/ractor_core.h @@ -13,7 +13,7 @@ #define RUBY_TYPED_FROZEN_SHAREABLE_NO_REC RUBY_FL_FINALIZE /* An in-flight move payload, serialized off-heap (defined in ractor.c). */ -struct rb_ractor_move_courier; +struct rb_ractor_courier; struct rb_ractor_sync { // ractor lock @@ -38,6 +38,11 @@ struct rb_ractor_sync { struct st_table *ports; size_t next_port_id; + /* The baskets this Ractor holds that are on no queue: one it is building to send, + * and one it has taken off a queue and is materializing. A queued basket is rooted + * by its queue instead. Only the owner touches this list. */ + struct ccan_list_head off_queue_baskets; + // monitors struct ccan_list_head monitors; @@ -46,23 +51,10 @@ struct rb_ractor_sync { VALUE legacy; bool legacy_exc; bool legacy_taken; /* Ractor#value already returned the value */ - - /* Number of receives currently materializing a copy (only the owner's threads - * update it, under the GVL). */ - int materializing_copies; }; struct ractor_basket; -/* One in-flight copy payload being rebuilt (lives on the receiver's machine - * stack) */ -struct ractor_materialize_frame { - VALUE snapshot; /* the sender-side snapshot */ - const VALUE *pinned; /* pin list of every snapshot node (owned by the basket) */ - size_t pinned_cnt; - struct ractor_materialize_frame *prev; -}; - // created // | ready to run // ====================== inserted to vm->ractor @@ -162,28 +154,11 @@ struct rb_ractor_struct { * still enumerates it. */ void *creating_child_objspace; - /* True while Ractor#send builds a native copy snapshot; copy_enter then collects - * every snapshot node into pin_capture below. Owner thread only. */ - bool gen_fields_capturing; - - /* Pin list collecting every node while a copy snapshot is built (basket_new - * hands it over to the basket). A global GC clears every shref, so the re-pin - * has to cover all nodes, not just the root. */ - VALUE *pin_capture; - size_t pin_capture_cnt, pin_capture_capa; - /* The in-flight copy basket between basket_new and the enqueue, so the re-pin - * covers that window too */ - struct ractor_basket *sending_basket; }; // rb_ractor_t is defined in vm_core.h /* Mark the GC roots held in Ractor r's C structs (from the root scan in gc.c). */ void rb_ractor_mark_local_roots(rb_ractor_t *r); void rb_ractor_mark_terminated_join_value(rb_ractor_t *r); -void rb_ractor_repin_in_flight(rb_ractor_t *r); -void rb_ractor_mark_in_flight_for_single_objspace(rb_ractor_t *r); -/* True while the current Ractor is materializing an arriving copy (see the - * definition in ractor_sync.c). */ -bool rb_ractor_materializing_p(void); /* Move src's registered_marks to dst and leave src empty (on join or when an orphan * is absorbed). An absorb can run during a GC sweep, so the implementation uses raw diff --git a/ractor_sync.c b/ractor_sync.c index 13cbc6ad3f136a..74a3daec193559 100644 --- a/ractor_sync.c +++ b/ractor_sync.c @@ -19,10 +19,14 @@ static struct ractor_basket *ractor_basket_new_ref(VALUE shareable); static void ractor_send_basket(rb_execution_context_t *ec, const struct ractor_port *rp, struct ractor_basket *b, bool raise_on_error); static void ractor_add_port(rb_ractor_t *r, st_data_t id); -// The off-heap courier used for moves. It is defined in ractor.c. -struct rb_ractor_move_courier *rb_ractor_move_courier_build(VALUE obj); -VALUE rb_ractor_move_courier_materialize(struct rb_ractor_move_courier *c); -void rb_ractor_move_courier_free(struct rb_ractor_move_courier *c); +// The off-heap courier a copy or a move payload travels in. Defined in ractor.c. +struct rb_ractor_courier *rb_ractor_courier_build_move(VALUE obj, struct rb_ractor_courier **slot); +VALUE rb_ractor_courier_materialize(struct rb_ractor_courier *c); +void rb_ractor_courier_free(struct rb_ractor_courier *c); +static void ractor_off_queue_add(rb_ractor_t *cr, struct ractor_basket *b); +static void ractor_off_queue_remove(struct ractor_basket *b); +void rb_ractor_courier_mark(struct rb_ractor_courier *c); +struct rb_ractor_courier *rb_ractor_courier_build_copy(VALUE obj, struct rb_ractor_courier **slot); static void ractor_port_mark(void *ptr) @@ -229,17 +233,18 @@ struct ractor_basket { * Marshal byte String. The receiver rebuilds it with Marshal.load instead * of walking it natively. */ bool marshaled; - /* The off-heap (xmalloc) courier of a basket_type_move. A move basket does - * not use v. */ - struct rb_ractor_move_courier *move_courier; - /* Every node of a native copy snapshot, collected while building it (raw - * malloc). The global GC's re-pin walks this list, since traversing the graph - * in-GC would need generic-ivar lookups. NULL: only the root (p.v) is pinned. */ - VALUE *pinned; - size_t pinned_cnt; + /* The off-heap (xmalloc) courier the payload graph was serialized into. + * Copy and move both use it; when set, v is unused. */ + struct rb_ractor_courier *courier; + /* The marshaled bytes of a copy payload, off-heap like the courier. When + * set, v is unused: an in-flight payload that is not a GC object needs no + * in-flight pin, so it never keeps a page of the sender's heap alive. */ + char *mbuf; + size_t mlen; } p; // payload - struct ccan_list_node node; + struct ccan_list_node node; /* the port queue it waits on */ + struct ccan_list_node off_queue_node; /* or sync.off_queue_baskets, when on none */ }; #if 0 @@ -259,10 +264,15 @@ ractor_basket_none_p(const struct ractor_basket *b) static void ractor_basket_mark(const struct ractor_basket *b) { - /* A move courier lives off-heap, and the shareable REFs it carries are marked and - * pinned as a global GC root by the in-flight registry (ractor.c). Nothing to do - * here. */ - if (b->type != basket_type_move) { + if (b->p.courier != NULL) { + /* The payload became this Ractor's to root the moment the message was enqueued + * here: the sender's own roots stop at the send. Before and after the queue the + * basket is on its holder's off_queue_baskets instead, so a courier is rooted + * from the moment it is allocated to the moment it is freed. */ + rb_ractor_courier_mark(b->p.courier); + } + else if (b->p.mbuf == NULL) { + /* Marshaled bytes are off-heap and hold nothing to mark. */ rb_gc_mark(b->p.v); } } @@ -270,19 +280,14 @@ ractor_basket_mark(const struct ractor_basket *b) static void ractor_basket_free(struct ractor_basket *b) { - /* A basket that dies before being enqueued clears the sender's re-pin slot; a - * free by the Ractor tearing the queue down does not match and is a no-op. */ - rb_ractor_t *cr = rb_current_ractor_raw(false); - if (cr != NULL && cr->sending_basket == b) { - cr->sending_basket = NULL; - } - free(b->p.pinned); - b->p.pinned = NULL; - b->p.pinned_cnt = 0; - if (b->type == basket_type_move && b->p.move_courier) { - /* A move courier that was never consumed (a queue being torn down, say). */ - rb_ractor_move_courier_free(b->p.move_courier); - b->p.move_courier = NULL; + ractor_off_queue_remove(b); + ruby_xfree(b->p.mbuf); + b->p.mbuf = NULL; + b->p.mlen = 0; + if (b->p.courier) { + /* A courier that was never consumed (a queue being torn down, say). */ + rb_ractor_courier_free(b->p.courier); + b->p.courier = NULL; } SIZED_FREE(b); } @@ -291,9 +296,47 @@ static struct ractor_basket * ractor_basket_alloc(void) { struct ractor_basket *b = ALLOC(struct ractor_basket); + + /* Empty and mark-safe from the start: a basket goes on its holder's in-flight list + * before it has a payload, so a GC can walk it while it is still being filled. */ + b->type = basket_type_none; + b->sender = Qnil; + b->port_id = 0; + b->p.v = Qnil; + b->p.exception = false; + b->p.marshaled = false; + b->p.courier = NULL; + b->p.mbuf = NULL; + b->p.mlen = 0; + ccan_list_node_init(&b->off_queue_node); + return b; } +/* A basket is rooted by whoever holds it: a port queue while it waits there, and its + * holder's off-queue list while it is being built or materialized. */ +static void +ractor_off_queue_add(rb_ractor_t *cr, struct ractor_basket *b) +{ + VM_ASSERT(cr == rb_current_ractor_raw(false)); + ccan_list_add_tail(&cr->sync.off_queue_baskets, &b->off_queue_node); +} + +static void +ractor_off_queue_remove(struct ractor_basket *b) +{ + ccan_list_del_init(&b->off_queue_node); +} + +static void +ractor_mark_off_queue_baskets(rb_ractor_t *r) +{ + struct ractor_basket *b; + ccan_list_for_each(&r->sync.off_queue_baskets, b, off_queue_node) { + ractor_basket_mark(b); + } +} + // ractor-internal - ractor_queue struct ractor_queue { @@ -751,8 +794,6 @@ ractor_sync_mark(rb_ractor_t *r) rb_gc_mark(r->sync.default_port_value); - /* (A copy snapshot being materialized is not marked here: each EC's frame - * chain roots it in rb_execution_context_mark, which also re-pins it.) */ /* Until the value is absorbed this is its only reliable root (Qundef while the * Ractor still runs); after Ractor#value returns it, the Ruby side roots it. */ rb_gc_mark(r->sync.legacy); @@ -770,72 +811,17 @@ ractor_sync_mark(rb_ractor_t *r) ractor_mark_monitors(r); } if (!world_stopped) RACTOR_UNLOCK_SELF(r); - } -} - -/* Re-pin a copy basket's payload: the root and every collected node. */ -static void -ractor_basket_repin_in_flight(const struct ractor_basket *b) -{ - if (b->type != basket_type_copy) return; - rb_gc_pin_in_flight_message(b->p.v); - for (size_t i = 0; i < b->p.pinned_cnt; i++) { - rb_gc_pin_in_flight_message(b->p.pinned[i]); - } -} - -static void -ractor_queue_repin_in_flight(const struct ractor_queue *rq) -{ - const struct ractor_basket *b; - ccan_list_for_each(&rq->set, b, node) { - /* A move basket carries an off-heap courier, so it has no shref to re-pin; - * ractor_basket_mark marks the shareable VALUEs it carries instead. */ - ractor_basket_repin_in_flight(b); - } -} - -static int -ractor_repin_ports_i(st_data_t key, st_data_t val, st_data_t data) -{ - ractor_queue_repin_in_flight((struct ractor_queue *)val); - return ST_CONTINUE; -} - -/* A global GC clears every shref bit, so all in-flight payloads have to be re-pinned - * before the unified mark. Runs on the driver, under the barrier. */ -void -rb_ractor_repin_in_flight(rb_ractor_t *r) -{ - if (r->sync.ports) { - ractor_queue_repin_in_flight(r->sync.recv_queue); - st_foreach(r->sync.ports, ractor_repin_ports_i, 0); - } - /* Baskets already built but not enqueued yet (in flight on the send path). */ - if (r->sending_basket != NULL) { - ractor_basket_repin_in_flight(r->sending_basket); - } - /* A snapshot still being built (from prepare_payload's walk until it moves into - * the basket). */ - for (size_t i = 0; i < r->pin_capture_cnt; i++) { - rb_gc_pin_in_flight_message(r->pin_capture[i]); - } - /* Snapshots being materialized are re-pinned from the EC frame chains instead - * (rb_execution_context_mark, which also covers a suspended fiber's EC). */ -} -/* A single-objspace impl (mmtk) has no pin or shref bits and no zombie_objspaces, so - * plain marking from the wrapper keeps these alive; the default GC covers the same set - * with its pins and its zombie scan. */ -void -rb_ractor_mark_in_flight_for_single_objspace(rb_ractor_t *r) -{ - rb_gc_mark(r->sync.legacy); - if (r->sending_basket != NULL) { - ractor_basket_mark(r->sending_basket); - } - for (size_t i = 0; i < r->pin_capture_cnt; i++) { - rb_gc_mark(r->pin_capture[i]); + /* The baskets on no queue: one being built to send, one being materialized. + * Walked in every collection, like the queues. What they hold is shareable, but + * "only a global GC frees a shareable" does not hold: pinned_roots_mark, which + * roots a shareable from its page bit, is skipped once the process is back to a + * single Ractor (rb_gc_single_objspace_p), and then an ordinary local GC frees + * one that nothing else names. A payload in flight is named by its basket and + * nothing else, so this list has to be a root whenever the queues are. No sync + * lock, though: only the owner touches it (the lock above guards the queues, + * which a foreign sender writes). */ + ractor_mark_off_queue_baskets(r); } } @@ -882,6 +868,7 @@ ractor_sync_init(rb_ractor_t *r) rb_native_mutex_initialize(&r->sync.lock); // monitors + ccan_list_head_init(&r->sync.off_queue_baskets); ccan_list_head_init(&r->sync.monitors); // waiters @@ -900,7 +887,6 @@ ractor_sync_init(rb_ractor_t *r) r->sync.legacy = Qundef; // no receive is rebuilding a payload yet - r->sync.materializing_copies = 0; #ifndef RUBY_THREAD_PTHREAD_H rb_native_cond_initialize(&r->sync.wakeup_cond); @@ -1017,7 +1003,8 @@ ractor_marshal_dump_rescue(VALUE obj, VALUE errinfo) } static VALUE -ractor_prepare_payload(rb_execution_context_t *ec, VALUE obj, enum ractor_basket_type *ptype, bool *pmarshaled) +ractor_prepare_payload(rb_execution_context_t *ec, VALUE obj, enum ractor_basket_type *ptype, bool *pmarshaled, + struct rb_ractor_courier **pcourier) { switch (*ptype) { case basket_type_ref: @@ -1029,37 +1016,18 @@ ractor_prepare_payload(rb_execution_context_t *ec, VALUE obj, enum ractor_basket } else { /* Snapshot the object on the sender side without calling the user-visible - * #clone: core types are deep-copied natively and anything else is - * marshaled here, so its user hooks run on the sender. */ + * #clone. Both forms are off-heap, so an in-flight payload is never a GC + * object and needs no pin: nothing of the sender's heap stays alive while + * the message waits (design_v2.md 4.5). The courier carries the core + * types; anything else is marshaled here, so its user hooks run on the + * sender, and the dump travels as plain bytes. */ *ptype = basket_type_copy; - /* During a native copy, copy_enter collects every snapshot node into the - * pin list that covers construction, enqueue and materialization. */ - rb_ractor_t *cr = rb_ec_ractor_ptr(ec); - VM_ASSERT(!cr->gen_fields_capturing); - cr->gen_fields_capturing = true; - VALUE snapshot = Qundef; - /* A native copy can raise (allocation, async interrupt). Leaving the - * capturing flag set would fail the next send's assert and leak a stale - * pin_capture list into that basket. */ - enum ruby_tag_type state; - EC_PUSH_TAG(ec); - if ((state = EC_EXEC_TAG()) == TAG_NONE) { - snapshot = ractor_copy_native_try(obj); - } - EC_POP_TAG(); - cr->gen_fields_capturing = false; - if (state != TAG_NONE) { - cr->pin_capture_cnt = 0; - EC_JUMP_TAG(ec, state); - } - if (UNDEF_P(snapshot)) { - cr->pin_capture_cnt = 0; - snapshot = rb_rescue2(ractor_marshal_dump_body, obj, - ractor_marshal_dump_rescue, obj, - rb_eTypeError, (VALUE)0); - *pmarshaled = true; - } - return snapshot; + if (rb_ractor_courier_build_copy(obj, pcourier) != NULL) return Qundef; + + *pmarshaled = true; + return rb_rescue2(ractor_marshal_dump_body, obj, + ractor_marshal_dump_rescue, obj, + rb_eTypeError, (VALUE)0); } } } @@ -1067,95 +1035,56 @@ ractor_prepare_payload(rb_execution_context_t *ec, VALUE obj, enum ractor_basket static struct ractor_basket * ractor_basket_new(rb_execution_context_t *ec, VALUE obj, enum ractor_basket_type type, bool exc) { - /* A copy payload's preparation can raise (an uncopyable object), so it runs before - * the basket is allocated and cannot leak one; the move branch allocates first, - * since an alloc raise must not orphan an already built courier. */ + rb_ractor_t *cr = rb_ec_ractor_ptr(ec); + /* Allocate and list the basket before anything is built into it: from here the + * courier it is about to hold is rooted by this Ractor's in-flight list, even + * half-built, and every raise below frees it through one path. */ + struct ractor_basket *b = ractor_basket_alloc(); + ractor_off_queue_add(cr, b); + volatile VALUE v = Qfalse; bool marshaled = false; - struct rb_ractor_move_courier *courier = NULL; + char *mbuf = NULL; + size_t mlen = 0; - struct ractor_basket *b; - if (type == basket_type_move) { - /* Allocate the basket first: its xmalloc can raise NoMemoryError, and a courier - * already built (sources destroyed, registry entry live) would be orphaned. */ - b = ractor_basket_alloc(); - enum ruby_tag_type state; - EC_PUSH_TAG(ec); - if ((state = EC_EXEC_TAG()) == TAG_NONE) { + enum ruby_tag_type state; + EC_PUSH_TAG(ec); + if ((state = EC_EXEC_TAG()) == TAG_NONE) { + if (type == basket_type_move) { /* Serialize the graph into an off-heap courier; the sources become * RactorMovedObject. While in flight there is no GC object left for the - * sender's GC to mark, sweep or move. */ - courier = rb_ractor_move_courier_build(obj); + * sender's GC to mark, sweep or move. The build publishes the courier into + * the basket as soon as it exists. */ + rb_ractor_courier_build_move(obj, &b->p.courier); } - EC_POP_TAG(); - if (state != TAG_NONE) { - SIZED_FREE(b); - EC_JUMP_TAG(ec, state); + else { + v = ractor_prepare_payload(ec, obj, &type, &marshaled, &b->p.courier); + if (type == basket_type_copy && marshaled) { + /* Take the dump off-heap: the sender's copy of it is ordinary garbage + * from here, so nothing of its heap is held while the message waits. */ + mlen = (size_t)RSTRING_LEN(v); + mbuf = ALLOC_N(char, mlen > 0 ? mlen : 1); + memcpy(mbuf, RSTRING_PTR(v), mlen); + v = Qundef; + } } } - else { - v = ractor_prepare_payload(ec, obj, &type, &marshaled); - enum ruby_tag_type state; - EC_PUSH_TAG(ec); - if ((state = EC_EXEC_TAG()) == TAG_NONE) { - b = ractor_basket_alloc(); - } - EC_POP_TAG(); - if (state != TAG_NONE) { - /* Drop the pin list, or every global GC re-pins the dead snapshot from it - * forever (rb_ractor_repin_in_flight walks it unconditionally). The nodes - * stay shref-pinned only until the next global GC clears the bits. */ - rb_ractor_t *cr = rb_ec_ractor_ptr(ec); - free(cr->pin_capture); - cr->pin_capture = NULL; - cr->pin_capture_cnt = cr->pin_capture_capa = 0; - EC_JUMP_TAG(ec, state); - } - /* copy_enter pinned every node at construction with cr->pin_capture as the - * re-pin source; hand it to the basket only after basket_alloc (which may GC) - * so the cover never lapses. A marshaled String is pinned here, after the - * alloc, so an alloc raise leaves no stale pin. */ - if (type == basket_type_copy && marshaled) { - rb_gc_pin_in_flight_message(v); - } + EC_POP_TAG(); + if (state != TAG_NONE) { + ruby_xfree(mbuf); + ractor_basket_free(b); /* leaves the list and frees a courier already built */ + EC_JUMP_TAG(ec, state); } b->type = type; b->p.exception = exc; b->p.v = v; b->p.marshaled = marshaled; - b->p.move_courier = courier; - b->p.pinned = NULL; - b->p.pinned_cnt = 0; - if (type == basket_type_copy) { - /* Hand the pin list to the basket, moving the re-pin cover from - * cr->pin_capture to cr->sending_basket with no safepoint in between. */ - rb_ractor_t *cr = rb_ec_ractor_ptr(ec); - b->p.pinned = cr->pin_capture; - b->p.pinned_cnt = cr->pin_capture_cnt; - VM_ASSERT(cr->sending_basket == NULL); - cr->sending_basket = b; - cr->pin_capture = NULL; - cr->pin_capture_cnt = cr->pin_capture_capa = 0; - } + b->p.mbuf = mbuf; + b->p.mlen = mlen; return b; } -/* True while this Ractor materializes an arriving copy: the half-built result - * legitimately points at the sender-resident (pinned) snapshot, so a local GC's - * verifier must not report containment violations, and the copy's own allocations can - * start that GC. */ -bool -rb_ractor_materializing_p(void) -{ - const rb_ractor_t *cr = rb_current_ractor_raw(false); - if (cr == NULL) return false; - /* Only a COPY materialization sets this: move shells reference other shells in - * this objspace, never the sender's graph. The count is per Ractor, so a fiber - * switch keeps it exact. */ - return cr->sync.materializing_copies > 0; -} - static VALUE ractor_basket_value(struct ractor_basket *b) { @@ -1163,40 +1092,26 @@ ractor_basket_value(struct ractor_basket *b) case basket_type_ref: break; case basket_type_copy: { - /* Materialize the sender's snapshot into the receiving Ractor's objspace. - * Passing the sender-resident graph by reference would create an unshareable - * cross-objspace edge that neither local GC can follow. The snapshot stays - * pinned in the sender's objspace and becomes garbage there once this copy - * finishes. Marshal.load allocates through this Ractor's normal newobj and - * write-barrier paths. - * - * Rebuilding can raise (marshal load hooks and autoload run user code and an - * async interrupt can arrive anywhere), and those hooks can run a nested - * Ractor.receive. The frame is pushed on the machine stack and popped under a - * TAG, so the chain never leaks a dead materialization or drops an outer one. */ + /* An off-heap copy courier rebuilds exactly like a move one; only the sources + * differ (still alive here, already shells there). */ + if (b->p.courier != NULL) goto materialize_courier; + /* The payload is the marshaled bytes. Marshal.load allocates through this + * Ractor's normal newobj and write-barrier paths, and can raise (load hooks and + * autoload run user code, an async interrupt can arrive anywhere), so it runs + * under a TAG. */ rb_execution_context_t *ec = rb_current_ec_noinline(); - rb_ractor_t *cr = rb_ec_ractor_ptr(ec); - struct ractor_materialize_frame frame = { - .snapshot = b->p.v, .pinned = b->p.pinned, .pinned_cnt = b->p.pinned_cnt, - .prev = ec->materialize_frames, - }; - ec->materialize_frames = &frame; - cr->sync.materializing_copies++; VALUE result = Qundef; enum ruby_tag_type state; EC_PUSH_TAG(ec); if ((state = EC_EXEC_TAG()) == TAG_NONE) { - if (b->p.marshaled) { - result = rb_marshal_load(b->p.v); - } - else { - result = ractor_copy_native_try(b->p.v); - if (UNDEF_P(result)) rb_bug("ractor_basket_value: native snapshot not natively copyable"); - } + /* Rebuild the byte string in this Ractor's objspace. Marshal does not mark + * its source (mark_load_arg) and the basket is off the queue, so this + * frame's stack slot is the String's only root for the load. */ + VALUE bin = rb_str_new(b->p.mbuf, (long)b->p.mlen); + result = rb_marshal_load(bin); + RB_GC_GUARD(bin); } EC_POP_TAG(); - ec->materialize_frames = frame.prev; - cr->sync.materializing_copies--; /* rb_copy_generic_ivar left the sender-resident snapshot host and fields_obj in * this EC's gen_fields_cache; the snapshot is garbage on the sender now, and a * stale cache hit on a reused address would deref a freed foreign fields_obj. @@ -1214,19 +1129,20 @@ ractor_basket_value(struct ractor_basket *b) RB_GC_GUARD(result); break; } - case basket_type_move: { + case basket_type_move: + materialize_courier: { /* Rebuild the moved graph from the off-heap courier into this Ractor's * objspace. The sources are already RactorMovedObject (set when the courier * was built), so move's snapshot semantics hold. The courier is xmalloc'd * rather than a GC object, so the sender's concurrent local GC never touches - * it; the VALUEs it carries are shareable or immediates, marked and pinned as - * a global GC root by the in-flight registry (ractor.c). + * it; the shareable VALUEs it carries are marked through this basket, which is + * on this Ractor's off-queue list until it is freed. * * Rebuilding can raise here too (rb_hash_aset on a moved key with a custom * #hash runs user code, and an async interrupt can arrive). On a raise the * courier is still owned by the basket, whose teardown frees it. */ rb_execution_context_t *ec = rb_current_ec_noinline(); - struct rb_ractor_move_courier *courier = b->p.move_courier; + struct rb_ractor_courier *courier = b->p.courier; /* Keep the materialized graph on the machine stack (result): it is the only * root until it reaches the caller. courier_free below runs a long loop, and * only the malloc'd basket's p.v holding it would give a concurrent global GC a @@ -1235,16 +1151,16 @@ ractor_basket_value(struct ractor_basket *b) enum ruby_tag_type state; EC_PUSH_TAG(ec); if ((state = EC_EXEC_TAG()) == TAG_NONE) { - result = rb_ractor_move_courier_materialize(courier); + result = rb_ractor_courier_materialize(courier); } EC_POP_TAG(); if (state != TAG_NONE) { - /* An unconsumed courier stays in b->p.move_courier; basket_free frees it. */ + /* An unconsumed courier stays in b->p.courier; basket_free frees it. */ ractor_basket_free(b); EC_JUMP_TAG(ec, state); } - rb_ractor_move_courier_free(courier); - b->p.move_courier = NULL; + rb_ractor_courier_free(courier); + b->p.courier = NULL; b->p.v = result; RB_GC_GUARD(result); break; @@ -1552,6 +1468,8 @@ ractor_try_receive(rb_execution_context_t *ec, rb_ractor_t *cr, const struct rac } struct ractor_basket *b = ractor_queue_deq(cr, rq); + /* Off the queue and not yet freed: this Ractor roots it while it materializes. */ + if (b) ractor_off_queue_add(cr, b); if (rq->closed && ractor_queue_empty_p(cr, rq)) { ractor_delete_port(cr, ractor_port_id(rp), false); @@ -1601,16 +1519,9 @@ ractor_send_basket(rb_execution_context_t *ec, const struct ractor_port *rp, str } else { b->port_id = ractor_port_id(rp); + /* The receiver's queue roots it from here; drop it from ours. */ + ractor_off_queue_remove(b); ractor_queue_enq(rp->r, rp->r->sync.recv_queue, b); - /* From basket_new to the enqueue the sender's sending_basket slot covers - * the re-pin; from here the queue walk does, so drop the slot (no safepoint - * or malloc-triggered GC inside the lock, so the cover never lapses). */ - if (b->type == basket_type_copy) { - rb_ractor_t *scr = rb_current_ractor_raw(false); - if (scr != NULL && scr->sending_basket == b) { - scr->sending_basket = NULL; - } - } } } RACTOR_UNLOCK(rp->r); @@ -1623,8 +1534,11 @@ ractor_send_basket(rb_execution_context_t *ec, const struct ractor_port *rp, str else { RUBY_DEBUG_LOG("closed:%u@r%u", (unsigned int)ractor_port_id(rp), rb_ractor_id(rp->r)); + /* Nothing took the basket: it was not enqueued, so free it whether or not the + * caller wants the error raised. */ + ractor_basket_free(b); + if (raise_on_error) { - ractor_basket_free(b); rb_raise(rb_eRactorClosedError, "The port was already closed"); } } @@ -1643,9 +1557,9 @@ ractor_basket_new_ref(VALUE shareable) b->p.v = shareable; b->p.exception = false; b->p.marshaled = false; - b->p.move_courier = NULL; - b->p.pinned = NULL; - b->p.pinned_cnt = 0; + b->p.courier = NULL; + b->p.mbuf = NULL; + b->p.mlen = 0; return b; } diff --git a/re.c b/re.c index 332b850d10287c..81aa79f801cf5b 100644 --- a/re.c +++ b/re.c @@ -1084,7 +1084,7 @@ match_set_regs(VALUE match, int num_regs, const OnigPosition *beg, const OnigPos * registers are written out to an onig-independent blob so the original malloc'd area can be * freed, leaving an empty shell behind, and rebuilt from the blob on the receiving side. */ void * -rb_match_move_dump(VALUE match, VALUE *regexp_out, VALUE *str_out, int *num_regs_out) +rb_match_blob_dump(VALUE match, VALUE *regexp_out, VALUE *str_out, int *num_regs_out, bool release_source) { struct RMatch *rm = RMATCH(match); int n = rm->num_regs; @@ -1100,27 +1100,30 @@ rb_match_move_dump(VALUE match, VALUE *regexp_out, VALUE *str_out, int *num_regs blob[2 * i + 1] = end[i]; } - if (FL_TEST_RAW(match, RMATCH_ONIG)) { - onig_region_free(&rm->as.onig, 0); - memset(&rm->as.onig, 0, sizeof(rm->as.onig)); - FL_UNSET_RAW(match, RMATCH_ONIG); - } - if (rm->char_offset) { - ruby_xfree(rm->char_offset); - rm->char_offset = NULL; - rm->char_offset_num_allocated = 0; + /* A copy leaves the source usable; only a move takes its internals apart. */ + if (release_source) { + if (FL_TEST_RAW(match, RMATCH_ONIG)) { + onig_region_free(&rm->as.onig, 0); + memset(&rm->as.onig, 0, sizeof(rm->as.onig)); + FL_UNSET_RAW(match, RMATCH_ONIG); + } + if (rm->char_offset) { + ruby_xfree(rm->char_offset); + rm->char_offset = NULL; + rm->char_offset_num_allocated = 0; + } } return blob; } VALUE -rb_match_move_alloc(VALUE klass, int num_regs) +rb_match_blob_alloc(VALUE klass, int num_regs) { return match_alloc_n(klass, num_regs); } void -rb_match_move_load(VALUE match, VALUE regexp, VALUE str, int num_regs, const void *blob_) +rb_match_blob_load(VALUE match, VALUE regexp, VALUE str, int num_regs, const void *blob_) { const OnigPosition *blob = blob_; struct RMatch *rm = RMATCH(match); @@ -1139,7 +1142,7 @@ rb_match_move_load(VALUE match, VALUE regexp, VALUE str, int num_regs, const voi } void -rb_match_move_free(void *blob) +rb_match_blob_free(void *blob) { ruby_xfree(blob); } diff --git a/string.c b/string.c index 5481705cf8b3b4..09384e4470b1c7 100644 --- a/string.c +++ b/string.c @@ -1210,6 +1210,23 @@ rb_str_new_static(const char *ptr, long len) return str_new_static(rb_cString, ptr, len, 0); } +/* Take an xmalloc'd buffer as the String's body without copying it; the String owns it + * from here and frees it like any other heap string. ptr must hold capa bytes plus the + * terminator for encindex, which is what a Ractor courier's string node carries. */ +VALUE +rb_str_new_owned(char *ptr, long len, long capa, int encindex) +{ + RUBY_DTRACE_CREATE_HOOK(STRING, len); + VALUE str = str_alloc_heap(rb_cString); + RSTRING(str)->len = len; + RSTRING(str)->as.heap.ptr = ptr; + /* Freed by size (STR_HEAP_SIZE = capa + terminator), so capa must describe the + * allocation the caller made, not just the bytes in use. */ + RSTRING(str)->as.heap.aux.capa = capa; + rb_enc_associate_index(str, encindex); + return str; +} + VALUE rb_usascii_str_new_static(const char *ptr, long len) { diff --git a/vm.c b/vm.c index ff1ec24b93138e..749fe5cc1a760c 100644 --- a/vm.c +++ b/vm.c @@ -1495,7 +1495,7 @@ env_copy(const VALUE *src_ep, VALUE read_only_variables) for (unsigned int j=0; jlocal_table_size; j++) { if (id == body->local_table[j]) { // check reassignment - if (body->lvar_states[j] == lvar_reassigned) { + if (iseq_lvar_state_get(body->lvar_states, j) == lvar_reassigned) { VALUE name = rb_id2str(id); VALUE msg = rb_sprintf("cannot make a shareable Proc because " "the outer variable '%" PRIsVALUE "' may be reassigned.", name); @@ -3905,23 +3905,6 @@ rb_execution_context_mark(const rb_execution_context_t *ec) rb_gc_mark(ec->local_storage_recursive_hash_for_trace); rb_gc_mark(ec->private_const_reference); - /* Snapshots of copy receives being materialized; off the queue, this is their only - * root. A snapshot is sender-resident, skipped as foreign by our local GC; the - * global GC marks it and re-pins its shrefs (its clear pass dropped all). Move - * couriers are covered by the in-flight registry instead (ractor.c). */ - for (const struct ractor_materialize_frame *f = ec->materialize_frames; f != NULL; f = f->prev) { - rb_gc_mark(f->snapshot); - if (f->snapshot && !RB_SPECIAL_CONST_P(f->snapshot) && rb_gc_during_global_gc_p()) { - /* Every node, not just the root: if compaction moved a snapshot node, - * the address-keyed generic_fields entries and the dedup table would - * break. */ - rb_gc_pin_in_flight_message(f->snapshot); - for (size_t i = 0; i < f->pinned_cnt; i++) { - rb_gc_pin_in_flight_message(f->pinned[i]); - } - } - } - rb_gc_mark_movable(ec->storage); } @@ -4843,8 +4826,6 @@ Init_BareVM(void) rb_native_mutex_initialize(&vm->ractor.sync.lock); rb_native_cond_initialize(&vm->ractor.sync.terminate_cond); rb_native_mutex_initialize(&vm->ractor.generic_fields_lock); - rb_native_mutex_initialize(&vm->ractor.move_courier_registry_lock); - ccan_list_head_init(&vm->ractor.move_courier_registry); rb_native_mutex_initialize(&vm->gc.registered_globals.lock); vm->gc.orphan_merge_pjob = POSTPONED_JOB_HANDLE_INVALID; diff --git a/vm_backtrace.c b/vm_backtrace.c index 573d671de1a88a..1c7ff0dad5eef4 100644 --- a/vm_backtrace.c +++ b/vm_backtrace.c @@ -867,6 +867,51 @@ rb_backtrace_dup(VALUE btobj) } +/* Copy a backtrace's frames into an off-heap blob for a Ractor copy courier. A frame + * only references shareable iseq / method-entry imemos, so the blob can carry them as + * they are. It has no compaction update hook, so rb_backtrace_blob_mark pins them + * (rb_gc_mark, not _movable) for as long as the message is in flight. */ +void * +rb_backtrace_blob_dump(VALUE btobj, int *size_out) +{ + rb_backtrace_t *bt; + TypedData_Get_Struct(btobj, rb_backtrace_t, &backtrace_data_type, bt); + + int size = bt->backtrace_size; + *size_out = size; + rb_backtrace_location_t *blob = ALLOC_N(rb_backtrace_location_t, size > 0 ? size : 1); + MEMCPY(blob, bt->backtrace, rb_backtrace_location_t, size); + return blob; +} + +VALUE +rb_backtrace_blob_load(const void *blob_, int size) +{ + const rb_backtrace_location_t *blob = blob_; + rb_backtrace_t *dst; + VALUE btobj = backtrace_alloc_capa(size, &dst); + + dst->backtrace_size = size; + MEMCPY(dst->backtrace, blob, rb_backtrace_location_t, size); + for (int i = 0; i < size; i++) { + const rb_backtrace_location_t *fi = &dst->backtrace[i]; + if (fi->cme) RB_OBJ_WRITTEN(btobj, Qundef, (VALUE)fi->cme); + if (fi->iseq) RB_OBJ_WRITTEN(btobj, Qundef, (VALUE)fi->iseq); + } + /* strary / locary stay unset: the receiver rebuilds them lazily. */ + return btobj; +} + +void +rb_backtrace_blob_mark(const void *blob_, int size) +{ + const rb_backtrace_location_t *blob = blob_; + for (int i = 0; i < size; i++) { + if (blob[i].cme) rb_gc_mark((VALUE)blob[i].cme); + if (blob[i].iseq) rb_gc_mark((VALUE)blob[i].iseq); + } +} + static long backtrace_size(const rb_execution_context_t *ec) { diff --git a/vm_core.h b/vm_core.h index 47c1382b26135f..e5700b11140f41 100644 --- a/vm_core.h +++ b/vm_core.h @@ -410,6 +410,12 @@ enum rb_builtin_attr { typedef VALUE (*rb_jit_func_t)(struct rb_execution_context_struct *, struct rb_control_frame_struct *); typedef VALUE (*rb_zjit_func_t)(struct rb_execution_context_struct *, struct rb_control_frame_struct *, rb_jit_func_t); +enum lvar_state { + lvar_uninitialized, + lvar_initialized, + lvar_reassigned, +}; + struct rb_iseq_constant_body { enum rb_iseq_type type; @@ -507,11 +513,7 @@ struct rb_iseq_constant_body { const ID *local_table; /* must free */ - enum lvar_state { - lvar_uninitialized, - lvar_initialized, - lvar_reassigned, - } *lvar_states; + uint8_t *lvar_states; /* catch table */ struct iseq_catch_table *catch_table; @@ -735,12 +737,9 @@ typedef struct rb_vm_struct { #endif } sync; - /* VM-wide locks for the Ractor transfer/inheritance machinery, plus the - * registry of in-flight move couriers. All of them are leaf locks: no - * safepoint inside a critical section. */ + /* VM-wide locks for the Ractor transfer/inheritance machinery. All of them + * are leaf locks: no safepoint inside a critical section. */ rb_nativethread_lock_t generic_fields_lock; /* the shared generic-fields table in variable.c */ - struct ccan_list_head move_courier_registry; /* couriers in flight (ractor.c); the global GC marks them */ - rb_nativethread_lock_t move_courier_registry_lock; #ifdef RUBY_THREAD_PTHREAD_H // ractor scheduling @@ -1125,7 +1124,6 @@ struct rb_waiting_list { struct rb_fiber_struct *fiber; }; -struct ractor_materialize_frame; struct rb_execution_context_struct { /* execution information */ @@ -1178,11 +1176,6 @@ struct rb_execution_context_struct { VALUE fields_obj; } gen_fields_cache; - /* Chain of receive frames being materialized on this EC (LIFO; the frames live - * on the C stack). A thread or fiber switch cannot corrupt it, since each EC's - * chain only contains that EC's own nesting. */ - struct ractor_materialize_frame *materialize_frames; - /* for GC */ struct { VALUE *stack_start; diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index 32b76962b8262c..08d4181f77e1f6 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -1267,10 +1267,6 @@ pub struct rb_iseq_constant_body_iseq_insn_info { pub size: ::std::os::raw::c_uint, pub succ_index_table: *mut succ_index_table, } -pub const lvar_uninitialized: rb_iseq_constant_body_lvar_state = 0; -pub const lvar_initialized: rb_iseq_constant_body_lvar_state = 1; -pub const lvar_reassigned: rb_iseq_constant_body_lvar_state = 2; -pub type rb_iseq_constant_body_lvar_state = u32; #[repr(C)] pub struct rb_iseq_constant_body__bindgen_ty_1 { pub flip_count: rb_snum_t,