diff --git a/.ci/check-matrix-lists.sh b/.ci/check-matrix-lists.sh index 0d602abb..50115b54 100755 --- a/.ci/check-matrix-lists.sh +++ b/.ci/check-matrix-lists.sh @@ -68,6 +68,7 @@ test-shim-urandom-smp test-shim-urandom-toctou test-shim-urandom-wrap test-shim-cred-race +test-mmap-fastpath test-mremap-infra test-mremap-fork-tracking test-dev-shm-paths diff --git a/Makefile b/Makefile index 2affad1e..330f68c0 100644 --- a/Makefile +++ b/Makefile @@ -359,6 +359,33 @@ $(BUILD_DIR)/test-sigsuspend: tests/test-sigsuspend.c | $(BUILD_DIR) @echo " CROSS $< (with -lpthread)" $(Q)$(CROSS_COMPILE)gcc $(CROSS_TEST_CFLAGS) -Itests -o $@ $< -lpthread +# bench-mmap has a multi-threaded mmap_lock-contention section; needs -lpthread. +$(BUILD_DIR)/bench-mmap: tests/bench-mmap.c | $(BUILD_DIR) + @echo " CROSS $< (with -lpthread)" + $(Q)$(CROSS_COMPILE)gcc $(CROSS_TEST_CFLAGS) -o $@ $< -lpthread + +# test-mmap-lazy races concurrent first touch from several threads. +$(BUILD_DIR)/test-mmap-lazy: tests/test-mmap-lazy.c | $(BUILD_DIR) + @echo " CROSS $< (with -lpthread)" + $(Q)$(CROSS_COMPILE)gcc $(CROSS_TEST_CFLAGS) -o $@ $< -lpthread + +.PHONY: test-mmap-lazy +test-mmap-lazy: $(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-lazy + @$(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-lazy + @sh tests/test-mmap-dirty-stats.sh $(ELFUSE_BIN) \ + $(BUILD_DIR)/test-mmap-lazy + +# EL1 consumer-mmap integration/stress test. +$(BUILD_DIR)/test-mmap-fastpath: tests/test-mmap-fastpath.c | $(BUILD_DIR) + @echo " CROSS $< (with -lpthread)" + $(Q)$(CROSS_COMPILE)gcc $(CROSS_TEST_CFLAGS) -o $@ $< -lpthread + +.PHONY: test-mmap-fastpath +test-mmap-fastpath: $(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-fastpath + @$(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-fastpath + @sh tests/test-mmap-fastpath-stats.sh $(ELFUSE_BIN) \ + $(BUILD_DIR)/test-mmap-fastpath + # test-thread-churn creates >64 threads to force thread-table slot reuse. $(BUILD_DIR)/test-thread-churn: tests/test-thread-churn.c | $(BUILD_DIR) @echo " CROSS $< (with -lpthread)" diff --git a/docs/internals.md b/docs/internals.md index 6bfecb4d..cabb8e8d 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -524,6 +524,14 @@ goes above the structured area, never below. Post-push masking ### `mmap` Notes +Private anonymous mappings are lazy at 2 MiB materialization granularity. A +host-side hierarchical bitmap records which low-VA 2 MiB blocks contain any +valid TTBR0 PTE, independently of the dirty-block bitmap. `munmap` and recycled +fast-path arenas use this index to visit only materialized blocks, so untouched +multi-GiB reservations have length-independent teardown. When every mapping in +a per-vCPU arena has been released and the index confirms that no PTE remains, +the arena cursor rewinds in place instead of taking a refill HVC. + Aligned file-backed `MAP_SHARED` (fixed or non-fixed) installs a real host `mmap(MAP_FIXED|MAP_SHARED, fd)` overlay onto the guest slab so the kernel page cache keeps the mapping coherent with the file (and diff --git a/docs/usage.md b/docs/usage.md index 2ec20943..b6f75830 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -93,6 +93,16 @@ host `KEY=` imports as `KEY=`. An empty variable name is rejected. Given neither `--env` nor `--clear-env`, the guest inherits the host environment unchanged. `--clear-env` starts from nothing, leaving only what `--env` puts back. +### mmap call fast path + +The aarch64 EL1 consumer fast path is enabled by default for +`mmap(NULL, len, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, ...)` up to +32 GiB per request. +Set `ELFUSE_MMAP_FASTPATH=0` to disable it. Unsupported mmap shapes, exhausted +arenas, and full consumption rings fall back to the normal host syscall path. +Verbose tracing, the syscall histogram, GDB, and Rosetta keep mmap on the host +path so observability and translated-guest behavior are unchanged. + ## Common Launch Patterns Run a statically linked guest binary: diff --git a/mk/shim.mk b/mk/shim.mk index 235c635e..7e167170 100644 --- a/mk/shim.mk +++ b/mk/shim.mk @@ -1,11 +1,33 @@ -# EL1 kernel shim assembly pipeline +# EL1 kernel shim pipeline # -# shim.S -> shim.o -> shim.bin -> shim_blob.h (C byte array) +# shim.S + freestanding shim-mmap.c -> shim.o -> shim.bin -> shim_blob.h -$(BUILD_DIR)/shim.o: src/core/shim.S | $(BUILD_DIR) +# Disable RCpc codegen so acquire loads remain cumulative LDARs. The retire +# snapshot rule carries cross-vCPU causality through different atomic words; +# LDAPR is intentionally too weak for that protocol. +SHIM_CFLAGS := -O2 -Wall -Wextra -Wpedantic -Wshadow \ + -Wstrict-prototypes -Wmissing-prototypes -Wformat=2 \ + -Wimplicit-fallthrough -Wundef -Wnull-dereference \ + -Wno-unused-parameter -ffreestanding -fno-builtin \ + -fno-stack-protector -fno-unwind-tables \ + -fno-asynchronous-unwind-tables -mno-outline-atomics \ + -Xclang -target-feature -Xclang -rcpc +SHIM_LD ?= ld + +$(BUILD_DIR)/shim-asm.o: src/core/shim.S | $(BUILD_DIR) @echo " AS $<" $(Q)$(SHIM_AS) $(SHIM_ASFLAGS) -o $@ $< +$(BUILD_DIR)/shim-mmap.o: src/core/shim-mmap.c src/core/shim-mmap.h \ + src/core/mmap-fastpath.h src/core/shim-globals.h | $(BUILD_DIR) + @echo " CC $<" + $(Q)$(CC) $(SHIM_CFLAGS) -MMD -MP -MF $(BUILD_DIR)/shim-mmap.d \ + -Isrc -c -o $@ $< + +$(BUILD_DIR)/shim.o: $(BUILD_DIR)/shim-asm.o $(BUILD_DIR)/shim-mmap.o + @echo " LD $@" + $(Q)$(SHIM_LD) -static -arch arm64 -e _start -o $@ $^ + $(BUILD_DIR)/shim.bin: $(BUILD_DIR)/shim.o @echo " OBJCOPY $@" $(Q)$(OBJCOPY) -O binary $< $@ diff --git a/mk/verify.mk b/mk/verify.mk index c0fd241d..4f84bd46 100644 --- a/mk/verify.mk +++ b/mk/verify.mk @@ -340,6 +340,36 @@ VERIFY_ALIGN_SCAN := src/proved/align.h VERIFY_ALIGN_CLAIM := for ANY address, alignment, and search window VERIFY_ALIGN_UNPROVED := the region-array walk around them stays test-covered +# Includes align.h: request_fits calls align_up_ok and window_fits, so this +# proof must discharge their contracts too, not merely assume them, the same +# reason VERIFY_ELF appends VERIFY_UTILS_FCTS. +# +# MIN_GOALS is the complete count from a real run. Keep the floor at that count +# so a future edit that quietly drops a contract or a runtime-error obligation +# cannot turn a smaller proof into a pass. +# +# mmap_fastpath_pow2_clamped was originally the classic bit-smear +# round-up-to-power-of-two; that form's bound and power-of-two properties +# were confirmed unreachable by these provers (a single OR step already +# times out), the same wall align_up_ok's own history describes one level +# down. Rewritten to a doubling loop with an axiomatized power-of-two ghost +# invariant -- same inputs, same outputs, linear arithmetic instead of +# bitwise -- and it discharges completely; see the comment above it. +VERIFY_MMAPFASTPATH_SRC := src/proved/mmap-fastpath.h +VERIFY_MMAPFASTPATH_FCTS := mmap_fastpath_request_fits mmap_fastpath_pow2_clamped \ + mmap_fastpath_window_max mmap_fastpath_arena_size \ + align_up_ok window_fits +VERIFY_MMAPFASTPATH_MIN_GOALS ?= 93 +VERIFY_MMAPFASTPATH_MODEL := typed +VERIFY_MMAPFASTPATH_SCAN := src/proved/mmap-fastpath.h src/proved/align.h +VERIFY_MMAPFASTPATH_CLAIM := for ANY cursor/limit/len and ANY registration history +VERIFY_MMAPFASTPATH_UNPROVED := mmap_fastpath_window_max reporting an actual array member \ + rather than just an upper bound (the loop-invariant \ + preservation step for that claim times out even with a \ + ghost witness index, confirmed unreachable, see the comment \ + above it); the atomic control-block bookkeeping around all \ + four stays test-covered + VERIFY_PATHDEPTH_SRC := src/proved/pathdepth.h VERIFY_PATHDEPTH_FCTS := path_depth_push path_depth_pop VERIFY_PATHDEPTH_MIN_GOALS ?= 24 diff --git a/scripts/check-mutants.py b/scripts/check-mutants.py index 77c2de69..723f9029 100755 --- a/scripts/check-mutants.py +++ b/scripts/check-mutants.py @@ -131,6 +131,41 @@ def _load(stem, name): " return start <= limit && length <= limit - start;", " return start <= limit && length <= limit - start + 1;", ), + # ---- verify-mmapfastpath ---------------------------------------------- + ( + "mmapfastpath", + "src/proved/mmap-fastpath.h", + "mmap_fastpath_request_fits", + "accept every non-empty request (an allocation can pass the limit)", + " return window_fits(start, len, limit);", + " return true;", + ), + ( + "mmapfastpath", + "src/proved/mmap-fastpath.h", + "mmap_fastpath_pow2_clamped", + "clamp an oversized arena request to the lower bound", + " if (value >= MMAP_FAST_ARENA_MAX)\n" + " return MMAP_FAST_ARENA_MAX;", + " if (value >= MMAP_FAST_ARENA_MAX)\n" + " return MMAP_FAST_ARENA_MIN;", + ), + ( + "mmapfastpath", + "src/proved/mmap-fastpath.h", + "mmap_fastpath_window_max", + "discard a new maximum (the arena can be undersized)", + " if (window[i] > max)\n max = window[i];", + " if (window[i] > max)\n max = 0;", + ), + ( + "mmapfastpath", + "src/proved/mmap-fastpath.h", + "mmap_fastpath_arena_size", + "return an arena size above the configured maximum", + " return adaptive > covering ? adaptive : covering;", + " return UINT64_MAX;", + ), # ---- verify-cmsg ------------------------------------------------------- ( "cmsg", diff --git a/src/core/bootstrap.c b/src/core/bootstrap.c index 985d1a99..99fdbf3b 100644 --- a/src/core/bootstrap.c +++ b/src/core/bootstrap.c @@ -37,6 +37,7 @@ #include "syscall/signal.h" #include "debug/log.h" +#include "core/mmap-fastpath.h" /* Worst case: 7 fixed regions (shim, shim-data, vDSO, brk, stack, mmap RX, mmap * RW) plus up to ELF_MAX_SEGMENTS for both the executable and the interpreter. @@ -138,6 +139,26 @@ static void register_runtime_regions(guest_t *g, size_t shim_bin_len) guest_invalidate_ptes(g, 0, 0x1000); } +/* ELF/shim/stack bytes are populated directly in the slab before their page + * tables become live. Mark their semantic backing regardless of requested + * permissions: a read-only file segment is still nonzero and must be scrubbed + * if a later MAP_FIXED lazy-anonymous mapping reuses the same slab block. + * Synthetic page-table coverage for unallocated mmap space has no semantic + * region and therefore remains clean. + */ +static void mark_registered_backing_dirty(guest_t *g) +{ + for (int i = 0; i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->end <= r->start) + continue; + uint64_t len = r->end - r->start; + if (r->gpa_base > g->guest_size || len > g->guest_size - r->gpa_base) + continue; + guest_dirty_mark_range(g, r->gpa_base, r->gpa_base + len); + } +} + int guest_bootstrap_probe_elf(const char *elf_path, elf_info_t *info) { memset(info, 0, sizeof(*info)); @@ -302,7 +323,15 @@ static bool build_boot_regions(mem_region_t *regions, * to the vDSO page when splitting the block; otherwise vdso_build cannot * write into it through guest_ptr. */ - if (!append_boot_region(regions, nregions, g->shim_base, + + /* EL1 fast munmap walks and atomically clears the live TTBR0 tree. Give the + * page-table pool an identity VA visible only to EL1; EL0 remains unable to + * inspect or corrupt descriptors, and every guest syscall still rejects the + * encompassing infrastructure range. + */ + if (!append_boot_region(regions, nregions, g->pt_pool_base, g->pt_pool_end, + MEM_PERM_RW_EL1_ONLY) || + !append_boot_region(regions, nregions, g->shim_base, g->shim_base + shim_bin_len, MEM_PERM_RX) || /* shim_data is EL1-only: the guest must not directly read or write the @@ -563,6 +592,7 @@ int guest_bootstrap_prepare(guest_t *g, } register_runtime_regions(g, shim_bin_len); + mark_registered_backing_dirty(g); startup_trace_step("register_regions", t0); log_debug("TTBR0=0x%llx, IPA base=0x%llx", (unsigned long long) boot->ttbr0, @@ -756,6 +786,13 @@ int guest_bootstrap_create_vcpu(guest_t *g, */ shim_globals_set_singleton(g); + /* Publish the main vCPU's first arena only after shim_globals_init has + * cleared every recycled control slot. Verbose tracing keeps all shim + * syscall fast paths on HVC so the trace remains complete. + */ + if (!verbose) + mmap_fastpath_prepare_vcpu(g, current_thread); + HV_CHECK(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_CNTKCTL_EL1, CNTKCTL_EL1_EL0_TIMER_EN)); @@ -864,6 +901,7 @@ int guest_bootstrap_rosetta_post_reset(guest_t *g, g->rosetta_guest_base - g->rosetta_va_base, ROSETTA_PATH); register_runtime_regions(g, shim_bin_len); + mark_registered_backing_dirty(g); int rosetta_argc = 0; const char **rosetta_argv = NULL; diff --git a/src/core/guest.c b/src/core/guest.c index e6417d6e..b68cd666 100644 --- a/src/core/guest.c +++ b/src/core/guest.c @@ -41,13 +41,15 @@ #include #include "core/guest.h" +#include "core/mmap-fastpath.h" #include "proved/gva.h" #include "core/startup-trace.h" #include "debug/log.h" #include "utils.h" -#include "runtime/futex.h" /* futex_interrupt_request */ -#include "runtime/thread.h" /* thread_destroy_all_vcpus */ -#include "syscall/proc.h" /* proc_request_exit_group */ +#include "runtime/futex.h" /* futex_interrupt_request */ +#include "runtime/thread.h" /* thread_destroy_all_vcpus */ +#include "syscall/internal.h" /* mmap_lock (lazy fault-in) */ +#include "syscall/proc.h" /* proc_request_exit_group */ #include "syscall/signal.h" #include "syscall/wakeup-pipe.h" @@ -160,6 +162,7 @@ static const void *guest_host_memchr(const void *src, /* Forward declaration (defined in the page table section below) */ static int desc_to_perms(uint64_t desc); +static uint64_t *find_l2_entry(guest_t *g, uint64_t va); /* Page table pool allocator. */ @@ -356,6 +359,48 @@ static inline void pte_store_release(uint64_t *entry, uint64_t desc) memory_order_release); } +/* Low-VA TTBR0 occupancy index. All mutators run under mmap_lock (or during + * single-threaded bootstrap), so plain bitmap operations are sufficient. The + * page-table descriptors themselves remain release-published for lock-free + * guest walkers; this host-only index is never consulted by a vCPU. + */ +static inline bool guest_pte_present_index(uint64_t va, uint64_t *block_out) +{ + if (va >= GUEST_PTE_PRESENT_LIMIT) + return false; + *block_out = va / BLOCK_2MIB; + return true; +} + +static inline void guest_pte_present_set(guest_t *g, uint64_t va) +{ + uint64_t block; + if (!guest_pte_present_index(va, &block)) + return; + uint64_t word = block >> 6; + g->pte_present_blocks[word] |= 1ULL << (block & 63); + g->pte_present_summary[word >> 6] |= 1ULL << (word & 63); +} + +static inline void guest_pte_present_clear(guest_t *g, uint64_t va) +{ + uint64_t block; + if (!guest_pte_present_index(va, &block)) + return; + uint64_t word = block >> 6; + g->pte_present_blocks[word] &= ~(1ULL << (block & 63)); + if (g->pte_present_blocks[word] == 0) + g->pte_present_summary[word >> 6] &= ~(1ULL << (word & 63)); +} + +static inline bool guest_pte_present_test(const guest_t *g, uint64_t va) +{ + uint64_t block; + if (!guest_pte_present_index(va, &block)) + return false; + return (g->pte_present_blocks[block >> 6] & (1ULL << (block & 63))) != 0; +} + /* Public API */ /* FEAT_TLBIRANGE probe -- runs exactly once via pthread_once. ARMv8.4 @@ -577,6 +622,7 @@ int guest_init(guest_t *g, uint64_t size, uint32_t ipa_bits) */ g->segments[0] = (hvf_segment_t) {.ipa = GUEST_IPA_BASE, .len = size}; g->n_segments = 1; + pthread_cond_init(&g->materialize_cond, NULL); return 0; } @@ -689,6 +735,7 @@ int guest_init_from_shm(guest_t *g, */ g->segments[0] = (hvf_segment_t) {.ipa = GUEST_IPA_BASE, .len = size}; g->n_segments = 1; + pthread_cond_init(&g->materialize_cond, NULL); log_debug( "guest: CoW fork: mapped %llu GiB from shm " @@ -835,6 +882,7 @@ void guest_destroy(guest_t *g) close(g->shm_fd); g->shm_fd = -1; } + pthread_cond_destroy(&g->materialize_cond); } /* Check whether a candidate IPA range [gpa, gpa+size) overlaps the primary @@ -1264,9 +1312,11 @@ int guest_map_va_range(guest_t *g, * guest_split_block instead. Skip silently to mirror upstream's * sys_mmap_high_va "reuse existing GPA" behavior. */ + guest_pte_present_set(g, va); continue; } pte_store_release(&l2[l2_idx], make_block_desc(cur_gpa, perms)); + guest_pte_present_set(g, va); if (!bcast) { if (va < changed_lo) changed_lo = va; @@ -1586,11 +1636,11 @@ static uint64_t gva_contiguous_avail(const guest_t *g, * (MEM_PERM_R/W/X bitmask). The walk continues across adjacent L2/L3 entries * until a mapping, permission, or physical-contiguity break is found. */ -static void *gva_resolve_perm(const guest_t *g, - uint64_t gva, - uint64_t *avail, - int required_perms, - uint64_t avail_limit) +static void *gva_resolve_perm_walk(const guest_t *g, + uint64_t gva, + uint64_t *avail, + int required_perms, + uint64_t avail_limit) { /* Always walk page tables to enforce permissions. The guest slab is * identity-mapped (GVA == GPA == offset), but L2 block descriptors carry @@ -1651,14 +1701,114 @@ static void *gva_resolve_perm(const guest_t *g, return NULL; } +/* Host-access fault-in for lazy (deferred-PTE) regions. + * + * A syscall may target guest memory the guest itself has never touched: a fresh + * calloc()-style arena handed straight to read(2), a futex word inside an + * untouched mapping, an iovec into a new heap chunk. The guest-fault path + * (guest_materialize_lazy via the EL1 shim) never runs for those, so the + * page-table walk in gva_resolve_perm_walk fails even though the access is + * legal. Materialize the touched blocks here, then let the caller re-walk. + * + * Locking: takes mmap_lock; callers of the resolve API that already hold + * mmap_lock must use the _nofault variants. Do not call while holding any lock + * that ranks after mmap_lock in the ordering (syscall/internal.h); callers that + * resolve under such locks (futex bucket paths) pre-fault at their lock-free + * entry points so the hook never engages there. + * + * TLBI: guest_materialize_lazy accumulates TLBI requests in the calling + * thread's per-vCPU slot. On a vCPU thread the syscall epilogue emits them. On + * non-vCPU threads the request is lost, which is self-healing: a vCPU that + * still holds a stale negative TLB entry re-faults, and the already-valid + * early-return in guest_materialize_lazy re-issues a page TLBI for it without + * re-zeroing. + * + * Returns 0 if at least one block in [gva, gva+len) is now materialized (or + * already was), -1 if the range intersects no materializable lazy region. + */ +int guest_lazy_faultin_locked(const guest_t *cg, uint64_t gva, uint64_t len) +{ + /* The lazy machinery mutates page tables; the const on the resolve API + * reflects the pure-walk fast path, not this slow path. + */ + guest_t *g = (guest_t *) (uintptr_t) cg; + + if (gva >= g->guest_size) + return -1; /* High-VA / non-identity ranges are never lazy. */ + if (len == 0) + len = 1; + uint64_t end = (len > g->guest_size - gva) ? g->guest_size : gva + len; + + int rc = -1; + for (uint64_t b = gva & ~(uint64_t) (BLOCK_2MIB - 1); b < end; + b += BLOCK_2MIB) { + uint64_t probe = (b > gva) ? b : gva; + if (guest_materialize_lazy(g, probe) == 0) + rc = 0; + } + return rc; +} + +static int gva_lazy_faultin(const guest_t *cg, + uint64_t gva, + uint64_t len, + int required_perms) +{ + (void) required_perms; /* Region prot gates the retry walk, not this. */ + + int rc; + mmap_lock_acquire((guest_t *) (uintptr_t) cg); + rc = guest_lazy_faultin_locked(cg, gva, len); + mmap_lock_release(); + return rc; +} + +int guest_lazy_faultin(const guest_t *g, uint64_t gva, uint64_t len) +{ + return gva_lazy_faultin(g, gva, len, MEM_PERM_R); +} + +static void *gva_resolve_perm(const guest_t *g, + uint64_t gva, + uint64_t *avail, + int required_perms, + uint64_t avail_limit, + bool allow_faultin) +{ + void *ptr = + gva_resolve_perm_walk(g, gva, avail, required_perms, avail_limit); + if (!allow_faultin) + return ptr; + + /* Window the fault-in to what the caller actually needs. Length-less + * resolves (guest_ptr / guest_ptr_avail) materialize a single block; their + * callers iterate and re-enter here per chunk. + */ + uint64_t want = (avail_limit == UINT64_MAX) ? 1 : avail_limit; + if (!ptr) { + if (gva_lazy_faultin(g, gva, want, required_perms) < 0) + return NULL; + return gva_resolve_perm_walk(g, gva, avail, required_perms, + avail_limit); + } + if (avail && *avail < want && gva <= UINT64_MAX - *avail && + gva_lazy_faultin(g, gva + *avail, want - *avail, required_perms) == 0) { + void *again = + gva_resolve_perm_walk(g, gva, avail, required_perms, avail_limit); + if (again) + ptr = again; + } + return ptr; +} + void *guest_ptr(const guest_t *g, uint64_t gva) { - return gva_resolve_perm(g, gva, NULL, MEM_PERM_R, UINT64_MAX); + return gva_resolve_perm(g, gva, NULL, MEM_PERM_R, UINT64_MAX, true); } void *guest_ptr_w(const guest_t *g, uint64_t gva) { - return gva_resolve_perm(g, gva, NULL, MEM_PERM_W, UINT64_MAX); + return gva_resolve_perm(g, gva, NULL, MEM_PERM_W, UINT64_MAX, true); } void *guest_ptr_avail(const guest_t *g, @@ -1666,7 +1816,19 @@ void *guest_ptr_avail(const guest_t *g, uint64_t *avail, int required_perms) { - return gva_resolve_perm(g, gva, avail, required_perms, UINT64_MAX); + return gva_resolve_perm(g, gva, avail, required_perms, UINT64_MAX, true); +} + +/* Pure page-table walk without lazy fault-in. For callers that already hold + * mmap_lock or must avoid acquiring it (e.g. the stale-TLB re-walk in the EL0 + * fault handler, which runs after guest_materialize_lazy has been consulted). + */ +void *guest_ptr_avail_nofault(const guest_t *g, + uint64_t gva, + uint64_t *avail, + int required_perms) +{ + return gva_resolve_perm(g, gva, avail, required_perms, UINT64_MAX, false); } void *guest_ptr_bound(const guest_t *g, @@ -1675,7 +1837,7 @@ void *guest_ptr_bound(const guest_t *g, int required_perms, uint64_t len_limit) { - return gva_resolve_perm(g, gva, avail, required_perms, len_limit); + return gva_resolve_perm(g, gva, avail, required_perms, len_limit, true); } static inline int guest_copy(const guest_t *g, @@ -1683,7 +1845,8 @@ static inline int guest_copy(const guest_t *g, void *dst, const void *src, size_t len, - int required_perms) + int required_perms, + bool allow_faultin) { if (len == 0) return 0; @@ -1697,7 +1860,7 @@ static inline int guest_copy(const guest_t *g, while (copied < len) { uint64_t avail; void *ptr = gva_resolve_perm(g, gva + copied, &avail, required_perms, - (uint64_t) (len - copied)); + (uint64_t) (len - copied), allow_faultin); if (!ptr) return -1; size_t chunk = len - copied; @@ -1717,7 +1880,7 @@ static inline int guest_copy(const guest_t *g, int guest_read(const guest_t *g, uint64_t gva, void *dst, size_t len) { - return guest_copy(g, gva, dst, NULL, len, MEM_PERM_R); + return guest_copy(g, gva, dst, NULL, len, MEM_PERM_R, true); } int guest_read_small(const guest_t *g, uint64_t gva, void *dst, size_t len) @@ -1733,7 +1896,12 @@ int guest_read_small(const guest_t *g, uint64_t gva, void *dst, size_t len) int guest_write(guest_t *g, uint64_t gva, const void *src, size_t len) { - return guest_copy(g, gva, NULL, src, len, MEM_PERM_W); + return guest_copy(g, gva, NULL, src, len, MEM_PERM_W, true); +} + +int guest_write_nofault(guest_t *g, uint64_t gva, const void *src, size_t len) +{ + return guest_copy(g, gva, NULL, src, len, MEM_PERM_W, false); } size_t guest_write_partial(guest_t *g, @@ -1745,7 +1913,7 @@ size_t guest_write_partial(guest_t *g, while (done < len) { uint64_t avail; void *dst = gva_resolve_perm(g, gva + done, &avail, MEM_PERM_W, - (uint64_t) (len - done)); + (uint64_t) (len - done), true); if (!dst) return done; @@ -1789,7 +1957,7 @@ int guest_read_str(const guest_t *g, uint64_t gva, char *dst, size_t max) break; uint64_t avail; void *ptr = gva_resolve_perm(g, gva + copied, &avail, MEM_PERM_R, - (uint64_t) (limit - copied)); + (uint64_t) (limit - copied), true); if (!ptr) break; @@ -1879,6 +2047,7 @@ void guest_reset(guest_t *g) if (gpa > g->guest_size || len > g->guest_size - gpa) continue; /* backing lies outside the primary slab */ memset((uint8_t *) g->host_base + gpa, 0, len); + guest_dirty_clear_zeroed_range(g, gpa, gpa + len); } /* Zero page table pool (not tracked in region array) */ @@ -1915,6 +2084,8 @@ void guest_reset(guest_t *g) g->mmap_rw_gap_hint = 0; g->mmap_rx_gap_hint = 0; g->ttbr0 = 0; + memset(g->pte_present_blocks, 0, sizeof(g->pte_present_blocks)); + memset(g->pte_present_summary, 0, sizeof(g->pte_present_summary)); tlbi_request_clear(); g->elf_load_min = ELF_DEFAULT_BASE; @@ -2902,6 +3073,9 @@ uint64_t guest_build_page_tables(guest_t *g, const mem_region_t *regions, int n) { uint64_t base = g->ipa_base; + memset(g->pte_present_blocks, 0, sizeof(g->pte_present_blocks)); + memset(g->pte_present_summary, 0, sizeof(g->pte_present_summary)); + /* Allocate L0 table */ uint64_t l0_gpa = pt_alloc_page(g); if (!l0_gpa) @@ -2998,6 +3172,8 @@ uint64_t guest_build_page_tables(guest_t *g, const mem_region_t *regions, int n) * to the primary-buffer GPA where the bytes actually are. */ l2[l2_idx] = make_block_desc(output_ipa, block_perms); + if (lookup_addr >= base) + guest_pte_present_set(g, lookup_addr - base); } } @@ -3011,6 +3187,11 @@ uint64_t guest_build_page_tables(guest_t *g, const mem_region_t *regions, int n) if (!finalize_block_perms(g, regions, n)) return 0; + for (int r = 0; r < n; r++) { + if (regions[r].perms & MEM_PERM_W) + guest_dirty_mark_range(g, regions[r].gpa_start, regions[r].gpa_end); + } + guest_pt_gen_bump(g); return ttbr0; } @@ -3129,9 +3310,12 @@ int guest_extend_page_tables(guest_t *g, * an explicit PT_VALID test so the intent survives a future * descriptor-bit renumbering. */ - if (l2[l2_idx] & PT_VALID) + if (l2[l2_idx] & PT_VALID) { + guest_pte_present_set(g, addr); continue; + } pte_store_release(&l2[l2_idx], make_block_desc(ipa, perms)); + guest_pte_present_set(g, addr); if (!bcast) { if (addr < changed_lo) changed_lo = addr; @@ -3147,10 +3331,129 @@ int guest_extend_page_tables(guest_t *g, return 0; } +static uint64_t guest_va_next_page_table_block(const guest_t *g, + uint64_t va, + uint64_t end) +{ + if (!g || !g->ttbr0) + return end; + uint64_t base = g->ipa_base; + uint64_t *l0 = pt_at(g, g->ttbr0 - base); + if (!l0) + return end; + + va &= ~(uint64_t) (BLOCK_2MIB - 1); + while (va < end) { + uint64_t ipa = base + va; + unsigned l0_idx = (unsigned) (ipa / (512ULL * BLOCK_1GIB)); + if (l0_idx >= 512) + return end; + if (!(l0[l0_idx] & PT_VALID)) { + uint64_t next_ipa = (uint64_t) (l0_idx + 1) * 512ULL * BLOCK_1GIB; + if (next_ipa <= ipa || next_ipa - base <= va) + return end; /* wrap guard */ + va = next_ipa - base; + continue; + } + uint64_t *l1 = pt_at(g, (l0[l0_idx] & 0xFFFFFFFFF000ULL) - base); + if (!l1) + return end; + unsigned l1_idx = + (unsigned) ((ipa % (512ULL * BLOCK_1GIB)) / BLOCK_1GIB); + if (!(l1[l1_idx] & PT_VALID)) { + uint64_t next_ipa = (ipa / BLOCK_1GIB + 1) * BLOCK_1GIB; + if (next_ipa <= ipa || next_ipa - base <= va) + return end; + va = next_ipa - base; + continue; + } + return va; + } + return end; +} + +static uint64_t guest_va_next_indexed_block(const guest_t *g, + uint64_t va, + uint64_t end) +{ + if (va >= end || va >= GUEST_PTE_PRESENT_LIMIT) + return end; + if (end > GUEST_PTE_PRESENT_LIMIT) + end = GUEST_PTE_PRESENT_LIMIT; + + uint64_t first_block = va / BLOCK_2MIB; + uint64_t last_block = (end - 1) / BLOCK_2MIB; + uint64_t first_word = first_block >> 6; + uint64_t last_word = last_block >> 6; + uint64_t bits = + g->pte_present_blocks[first_word] & (~0ULL << (first_block & 63)); + if (first_word == last_word && (last_block & 63) != 63) + bits &= (1ULL << ((last_block & 63) + 1)) - 1; + if (bits) + return (first_word * 64 + (uint64_t) __builtin_ctzll(bits)) * + BLOCK_2MIB; + + uint64_t word = first_word + 1; + while (word <= last_word) { + uint64_t summary_word = word >> 6; + uint64_t summary_last = last_word >> 6; + uint64_t summary = + g->pte_present_summary[summary_word] & (~0ULL << (word & 63)); + if (summary_word == summary_last && (last_word & 63) != 63) + summary &= (1ULL << ((last_word & 63) + 1)) - 1; + if (summary) { + uint64_t present_word = + summary_word * 64 + (uint64_t) __builtin_ctzll(summary); + uint64_t present = g->pte_present_blocks[present_word]; + if (present_word == last_word && (last_block & 63) != 63) + present &= (1ULL << ((last_block & 63) + 1)) - 1; + if (present) + return (present_word * 64 + + (uint64_t) __builtin_ctzll(present)) * + BLOCK_2MIB; + } + if (summary_word == summary_last) + break; + word = (summary_word + 1) * 64; + } + return end; +} + +uint64_t guest_va_next_present_block(const guest_t *g, + uint64_t va, + uint64_t end) +{ + if (!g || va >= end) + return end; + if (va & (BLOCK_2MIB - 1)) { + if (va > UINT64_MAX - (BLOCK_2MIB - 1)) + return end; + va = ALIGN_2MIB_UP(va); + if (va >= end) + return end; + } + + if (va < GUEST_PTE_PRESENT_LIMIT) { + uint64_t low_end = + end < GUEST_PTE_PRESENT_LIMIT ? end : GUEST_PTE_PRESENT_LIMIT; + uint64_t next = guest_va_next_indexed_block(g, va, low_end); + if (next < low_end || end <= GUEST_PTE_PRESENT_LIMIT) + return next; + va = GUEST_PTE_PRESENT_LIMIT; + } + + /* Non-identity/high-VA mappings live outside the compact low-VA index. + * Preserve the page-table walker for those uncommon ranges. + */ + return guest_va_next_page_table_block(g, va, end); +} + bool guest_va_block_mapped(const guest_t *g, uint64_t va) { if (!g || !g->ttbr0 || (va & (BLOCK_2MIB - 1))) return false; + if (va < GUEST_PTE_PRESENT_LIMIT) + return guest_pte_present_test(g, va); uint64_t base = g->ipa_base; uint64_t *l0 = pt_at(g, g->ttbr0 - base); @@ -3179,6 +3482,71 @@ bool guest_va_block_mapped(const guest_t *g, uint64_t va) return (l2[l2_idx] & PT_VALID) != 0; } +void guest_rebuild_pte_present(guest_t *g) +{ + if (!g) + return; + memset(g->pte_present_blocks, 0, sizeof(g->pte_present_blocks)); + memset(g->pte_present_summary, 0, sizeof(g->pte_present_summary)); + if (!g->ttbr0) + return; + + uint64_t limit = g->guest_size; + if (limit > GUEST_PTE_PRESENT_LIMIT) + limit = GUEST_PTE_PRESENT_LIMIT; + for (uint64_t va = 0; va < limit; va += BLOCK_2MIB) { + uint64_t *l2_entry = find_l2_entry(g, va); + if (!l2_entry || !(*l2_entry & PT_VALID)) + continue; + if ((*l2_entry & 3) == 1) { + guest_pte_present_set(g, va); + continue; + } + uint64_t l3_ipa = *l2_entry & 0xFFFFFFFFF000ULL; + uint64_t *l3 = pt_at(g, l3_ipa - g->ipa_base); + if (!l3) + continue; + for (unsigned i = 0; i < BLOCK_2MIB / PAGE_SIZE; i++) { + if (l3[i] & PT_VALID) { + guest_pte_present_set(g, va); + break; + } + } + } +} + +void guest_retire_ptes_committed(guest_t *g, uint64_t start, uint64_t end) +{ + if (!g || end <= start) + return; + uint64_t block = ALIGN_2MIB_DOWN(start); + while (block < end) { + bool present = false; + uint64_t *l2_entry = find_l2_entry(g, block); + if (l2_entry && (*l2_entry & PT_VALID)) { + if ((*l2_entry & 3) == 1) { + present = true; + } else { + uint64_t l3_ipa = *l2_entry & 0xFFFFFFFFF000ULL; + uint64_t *l3 = pt_at(g, l3_ipa - g->ipa_base); + if (l3) { + for (unsigned i = 0; i < BLOCK_2MIB / PAGE_SIZE; i++) { + if (pte_load_acquire(&l3[i]) & PT_VALID) { + present = true; + break; + } + } + } + } + } + if (present) + guest_pte_present_set(g, block); + else + guest_pte_present_clear(g, block); + block += BLOCK_2MIB; + } +} + /* L3 page table splitting. */ /* L3 page descriptor: bits[1:0]=11 = valid page at level 3. This is distinct @@ -3335,6 +3703,7 @@ int guest_split_block(guest_t *g, uint64_t block_gpa) int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) { uint64_t base = g->ipa_base; + bool any_changed = false; /* Page-align the range. The ALIGN_UP step on end could wrap to 0 for inputs * within PAGE_SIZE-1 of UINT64_MAX, silently turning the invalidation into @@ -3349,10 +3718,21 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) return 0; for (uint64_t addr = start; addr < end;) { + uint64_t indexed_block = ALIGN_2MIB_DOWN(addr); + if (indexed_block < GUEST_PTE_PRESENT_LIMIT && + !guest_pte_present_test(g, indexed_block)) { + addr = + guest_va_next_present_block(g, indexed_block + BLOCK_2MIB, end); + continue; + } uint64_t *l2_entry = find_l2_entry(g, addr); if (!l2_entry) { - /* No L2 entry (already unmapped); skip this 2MiB block */ - addr = ALIGN_2MIB_UP(addr + 1); + /* No L2 table: nothing to invalidate in this block. The low-VA + * occupancy index jumps directly to the next block containing a + * valid PTE; high VA retains the page-table hierarchy fallback. + */ + guest_pte_present_clear(g, indexed_block); + addr = guest_va_next_present_block(g, ALIGN_2MIB_UP(addr + 1), end); continue; } @@ -3361,6 +3741,7 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) /* Not mapped at all: skip */ if (!(*l2_entry & 1)) { + guest_pte_present_clear(g, block_start); addr = block_end; continue; } @@ -3374,6 +3755,8 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) * broadcast. */ pte_store_release(l2_entry, 0); + guest_pte_present_clear(g, block_start); + any_changed = true; tlbi_request_range(base + block_start, base + block_end); addr = block_end; continue; @@ -3399,12 +3782,14 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) uint64_t page_end = (end < block_end) ? end : block_end; uint64_t changed_lo = UINT64_MAX, changed_hi = 0; bool bcast = tlbi_request_is_broadcast(); + bool block_changed = false; for (uint64_t pa = page_start; pa < page_end; pa += PAGE_SIZE) { unsigned l3_idx = (unsigned) (((base + pa) % BLOCK_2MIB) / PAGE_SIZE); if (l3[l3_idx] != 0) { pte_store_release(&l3[l3_idx], 0); /* Invalid descriptor */ + block_changed = true; if (!bcast) { if (pa < changed_lo) changed_lo = pa; @@ -3416,10 +3801,27 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) if (!bcast && changed_hi > changed_lo) tlbi_request_range(base + changed_lo, base + changed_hi); + if (block_changed) + any_changed = true; + + bool block_present = false; + if (page_start > block_start || page_end < block_end) { + for (unsigned i = 0; i < BLOCK_2MIB / PAGE_SIZE; i++) { + if (l3[i] & PT_VALID) { + block_present = true; + break; + } + } + } + if (block_present) + guest_pte_present_set(g, block_start); + else + guest_pte_present_clear(g, block_start); addr = page_end; } - guest_pt_gen_bump(g); + if (any_changed) + guest_pt_gen_bump(g); return 0; } @@ -3495,6 +3897,7 @@ int guest_update_perms(guest_t *g, uint64_t start, uint64_t end, int perms) pte_store_release(l2_entry, make_block_desc(ipa, perms)); tlbi_request_range(base + block_start, base + block_end); } + guest_pte_present_set(g, block_start); addr = block_end; continue; } @@ -3569,9 +3972,12 @@ int guest_update_perms(guest_t *g, uint64_t start, uint64_t end, int perms) if (!bcast && changed_hi > changed_lo) tlbi_request_range(base + changed_lo, base + changed_hi); + guest_pte_present_set(g, block_start); addr = page_end; } + if (perms & MEM_PERM_W) + guest_dirty_mark_range(g, start, end); guest_pt_gen_bump(g); return 0; } @@ -3650,19 +4056,136 @@ int guest_install_va_pages(guest_t *g, changed_hi = v + PAGE_SIZE; } } + guest_pte_present_set(g, v); } if (!bcast && changed_hi > changed_lo) tlbi_request_range(changed_lo, changed_hi); + if (perms & MEM_PERM_W) + guest_dirty_mark_range(g, gpa, gpa + length); guest_pt_gen_bump(g); return 0; } -/* Lazy page materialization for MAP_NORESERVE. */ +/* Lazy page materialization for deferred-PTE (private anonymous / + * MAP_NORESERVE) regions. + */ -int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) +bool guest_block_may_be_dirty(const guest_t *g, uint64_t block_start) +{ + if (!g || block_start >= g->guest_size) + return true; + uint64_t block = block_start / BLOCK_2MIB; + return (g->dirty_blocks[block >> 6] & (1ULL << (block & 63))) != 0; +} + +void guest_dirty_mark_range(guest_t *g, uint64_t start, uint64_t end) +{ + if (!g || end <= start || start >= g->guest_size) + return; + if (end > g->guest_size) + end = g->guest_size; + uint64_t first = start / BLOCK_2MIB; + uint64_t last = (end - 1) / BLOCK_2MIB; + for (uint64_t block = first; block <= last; block++) + g->dirty_blocks[block >> 6] |= 1ULL << (block & 63); +} + +void guest_dirty_clear_zeroed_range(guest_t *g, uint64_t start, uint64_t end) +{ + if (!g || end <= start || start >= g->guest_size) + return; + if (end > g->guest_size) + end = g->guest_size; + uint64_t first = ALIGN_2MIB_UP(start); + uint64_t last = ALIGN_2MIB_DOWN(end); + for (uint64_t addr = first; addr < last; addr += BLOCK_2MIB) { + uint64_t block = addr / BLOCK_2MIB; + g->dirty_blocks[block >> 6] &= ~(1ULL << (block & 63)); + } +} + +static bool materialize_claim_overlaps(const guest_materialize_claim_t *claim, + uint64_t start, + uint64_t end) +{ + return claim->active && start < claim->end && end > claim->start; +} + +void guest_materialize_wait_range_locked(guest_t *g, + uint64_t start, + uint64_t end) +{ + if (!g || end <= start) + return; + for (;;) { + bool overlap = false; + for (int i = 0; i < GUEST_MATERIALIZE_CLAIMS; i++) { + if (materialize_claim_overlaps(&g->materialize_claims[i], start, + end)) { + overlap = true; + break; + } + } + if (!overlap) + return; + mmap_lock_cond_wait(g, &g->materialize_cond); + } +} + +void guest_materialize_wait_all_locked(guest_t *g) { - /* Find the noreserve region containing this offset */ + guest_materialize_wait_range_locked(g, 0, UINT64_MAX); +} + +static int materialize_claim_alloc_locked(guest_t *g, + uint64_t start, + uint64_t end) +{ + guest_materialize_wait_range_locked(g, start, end); + for (int i = 0; i < GUEST_MATERIALIZE_CLAIMS; i++) { + guest_materialize_claim_t *claim = &g->materialize_claims[i]; + if (!claim->active) { + claim->start = start; + claim->end = end; + claim->active = true; + return i; + } + } + return -1; +} + +static void materialize_claim_release_locked(guest_t *g, int slot) +{ + if (slot < 0) + return; + g->materialize_claims[slot].active = false; + pthread_cond_broadcast(&g->materialize_cond); +} + +/* Whether the 4KiB page containing va has a valid stage-1 descriptor. Callers + * must hold mmap_lock; used to detect blocks a concurrent fault already + * materialized. + */ +bool guest_va_pte_valid(guest_t *g, uint64_t va) +{ + uint64_t *l2_entry = find_l2_entry(g, va); + if (!l2_entry || !(*l2_entry & PT_VALID)) + return false; + if ((*l2_entry & 3) == 1) + return true; /* 2MiB block descriptor */ + uint64_t *l3 = pt_at(g, (*l2_entry & 0xFFFFFFFFF000ULL) - g->ipa_base); + if (!l3) + return false; + unsigned l3_idx = + (unsigned) (((g->ipa_base + va) % BLOCK_2MIB) / PAGE_SIZE); + return (l3[l3_idx] & PT_VALID) != 0; +} + +static int guest_materialize_lazy_one(guest_t *g, uint64_t fault_offset) +{ +retry:; + /* Find the lazy region containing this offset */ const guest_region_t *region = NULL; for (int i = 0; i < g->nregions; i++) { if (g->regions[i].start <= fault_offset && @@ -3673,7 +4196,36 @@ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) } if (!region) - return -1; /* Not a noreserve region */ + return -1; /* Not a lazy region */ + + /* PROT_NONE is a reservation, not a mapping: a fault inside it is a genuine + * SIGSEGV, never a materialization request. Without this check a + * PROT_NONE|MAP_NORESERVE region would be silently granted read permission + * by the perms fallback below. + */ + if (region->prot == LINUX_PROT_NONE) + return -1; + + uint64_t block_start = fault_offset & ~(BLOCK_2MIB - 1); + uint64_t block_end = block_start + BLOCK_2MIB; + if (block_end > g->guest_size) + block_end = g->guest_size; + + /* Already materialized: another thread (concurrent guest fault or a + * host-side fault-in on a syscall path) completed this block while this + * vCPU was queued on mmap_lock, and the guest may have written real data + * through the new PTEs since. Running the memset below again would wipe + * those writes. The fault that got us here is then either a stale negative + * TLB entry or an in-flight retry. Invalidate the whole materialization + * block so this path follows the same one-RVAE-per-block contract as a + * newly installed block. + */ + if (guest_va_pte_valid(g, fault_offset)) { + mmap_fastpath_note_materialized_locked(g, block_start, block_end); + g->materialize_stats[GUEST_MATERIALIZE_ALREADY_VALID]++; + tlbi_request_range(g->ipa_base + block_start, g->ipa_base + block_end); + return 0; + } /* Materialize one 2MiB block containing the fault address. This is the * smallest granule that guest_extend_page_tables works with. For the common @@ -3681,12 +4233,18 @@ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) * trade-off: it avoids over-committing the large reservation while keeping * the fault rate manageable. */ - uint64_t block_start = fault_offset & ~(BLOCK_2MIB - 1); - uint64_t block_end = block_start + BLOCK_2MIB; - /* Clamp to guest size */ - if (block_end > g->guest_size) - block_end = g->guest_size; + /* A sibling may be zeroing another window in this block without the lock. + * Wait before inspecting regions/PTEs, then restart because a mutator that + * was itself waiting may have changed the region layout first. + */ + for (int i = 0; i < GUEST_MATERIALIZE_CLAIMS; i++) { + if (materialize_claim_overlaps(&g->materialize_claims[i], block_start, + block_end)) { + guest_materialize_wait_range_locked(g, block_start, block_end); + goto retry; + } + } uint64_t materialize_start = (block_start > region->start) ? block_start : region->start; @@ -3707,18 +4265,70 @@ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) if (perms == 0) perms = MEM_PERM_R; /* At minimum readable */ + /* Zero the window BEFORE any PTE becomes valid. The moment a descriptor is + * published, sibling vCPUs with no stale TLB entry can write through it + * without ever faulting; zeroing afterwards (the historical order) would + * wipe such a write. The slab is host memory, so zeroing needs no PTEs, and + * every writer that could touch the window first goes through mmap_lock + * (guest faults and host-side fault-in alike), so nothing can write between + * this memset and the descriptor stores below. Skip pages that are already + * valid: they belong to a previously materialized neighbor in the same + * block and may hold live data. + */ + int claim_slot = -1; + bool dirty = guest_block_may_be_dirty(g, block_start); + if (dirty) { + uint64_t zero_pages[8] = {0}; + bool any_valid = false; + for (uint64_t pg = materialize_start; pg < materialize_end; + pg += PAGE_SIZE) { + unsigned page = (unsigned) ((pg - block_start) / PAGE_SIZE); + if (guest_va_pte_valid(g, pg)) + any_valid = true; + else + zero_pages[page >> 6] |= 1ULL << (page & 63); + } + + claim_slot = materialize_claim_alloc_locked(g, block_start, block_end); + if (claim_slot >= 0) + mmap_lock_drop_keep_gate(); + for (unsigned page = 0; page < 512;) { + if (!(zero_pages[page >> 6] & (1ULL << (page & 63)))) { + page++; + continue; + } + unsigned first = page; + do { + page++; + } while (page < 512 && + (zero_pages[page >> 6] & (1ULL << (page & 63)))); + memset((uint8_t *) g->host_base + block_start + + (uint64_t) first * PAGE_SIZE, + 0, (uint64_t) (page - first) * PAGE_SIZE); + } + if (claim_slot >= 0) + mmap_lock_reacquire_with_gate(g); + if (!any_valid && materialize_start == block_start && + materialize_end == block_end) + guest_dirty_clear_zeroed_range(g, block_start, block_end); + } + /* Create page table entries. guest_extend_page_tables creates L2 block * descriptors but skips existing table descriptors (L2->L3 splits). * guest_update_perms handles the L3 case: if guest_invalidate_ptes * previously split the block and invalidated the L3 entries, update_perms * recreates them with correct perms. */ - if (guest_extend_page_tables(g, block_start, block_end, perms) < 0) + if (guest_extend_page_tables(g, block_start, block_end, perms) < 0) { + materialize_claim_release_locked(g, claim_slot); return -1; + } if (partial_block) { - if (guest_split_block(g, block_start) < 0) + if (guest_split_block(g, block_start) < 0) { + materialize_claim_release_locked(g, claim_slot); return -1; + } /* If this block had no page-table entry before the lazy fault, * guest_extend_page_tables() necessarily created a full 2MiB block. @@ -3728,25 +4338,103 @@ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) */ if (!had_mapping) { if (block_start < materialize_start && - guest_invalidate_ptes(g, block_start, materialize_start) < 0) + guest_invalidate_ptes(g, block_start, materialize_start) < 0) { + materialize_claim_release_locked(g, claim_slot); return -1; + } if (materialize_end < block_end && - guest_invalidate_ptes(g, materialize_end, block_end) < 0) + guest_invalidate_ptes(g, materialize_end, block_end) < 0) { + materialize_claim_release_locked(g, claim_slot); return -1; + } } } guest_update_perms(g, materialize_start, materialize_end, perms); - /* Zero the materialized memory. Only zero within the region boundaries to - * avoid clobbering adjacent data. - */ - if (materialize_end > materialize_start) - memset((uint8_t *) g->host_base + materialize_start, 0, - materialize_end - materialize_start); - - /* The page-table helpers above already requested the matching TLBI; no - * additional flush is needed here. + /* One 2MiB materialization gets one block-sized RVAE1IS. This is cheaper + * than per-page invalidation and covers every negative entry that may have + * been cached for the block while its descriptors were invalid. */ + tlbi_request_range(g->ipa_base + block_start, g->ipa_base + block_end); + g->materialize_stats[dirty ? GUEST_MATERIALIZE_DIRTY_MEMSET + : GUEST_MATERIALIZE_CLEAN_SKIP]++; + g->materialize_stats[GUEST_MATERIALIZE_WINDOW_BYTES] += + materialize_end - materialize_start; + mmap_fastpath_note_materialized_locked(g, materialize_start, + materialize_end); + materialize_claim_release_locked(g, claim_slot); return 0; } + +int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) +{ + return guest_materialize_lazy_one(g, fault_offset); +} + +int guest_materialize_lazy_fault(guest_t *g, uint64_t fault_offset) +{ + typedef struct { + guest_t *guest; + uint64_t next_block; + unsigned streak; + } fault_around_state_t; + static _Thread_local fault_around_state_t state; + + uint64_t block = ALIGN_2MIB_DOWN(fault_offset); + if (state.guest == g && block == state.next_block) { + if (state.streak < 4) + state.streak++; + } else { + state.guest = g; + state.streak = 0; + } + + const guest_region_t *region = guest_region_find(g, fault_offset); + if (!region || !region->noreserve || region->prot == LINUX_PROT_NONE) + return -1; + + unsigned blocks = 1U << state.streak; + if (blocks > 16) + blocks = 16; + uint64_t region_last = ALIGN_2MIB_UP(region->end); + if (region_last > g->guest_size) + region_last = g->guest_size; + + for (unsigned i = 0; i < blocks; i++) { + uint64_t ahead = block + (uint64_t) i * BLOCK_2MIB; + if (ahead >= region_last) + break; + if (guest_block_may_be_dirty(g, ahead) && blocks > 4) { + blocks = 4; + break; + } + } + + int result = -1; + uint64_t last = block; + for (unsigned i = 0; i < blocks; i++) { + uint64_t ahead = block + (uint64_t) i * BLOCK_2MIB; + if (ahead >= region_last) + break; + + /* The current block must probe the actual FAR. A fast-path mmap can + * extend an already materialized region within the same 2MiB block; + * probing the region/block start would see an older valid page and + * return without installing the newly faulted page. Ahead blocks have + * no FAR, so use their first address inside the region. + */ + uint64_t probe = (i == 0) + ? fault_offset + : (ahead < region->start ? region->start : ahead); + int rc = guest_materialize_lazy_one(g, probe); + if (i == 0) + result = rc; + if (rc < 0) + break; + last = ahead; + } + if (result == 0) + state.next_block = last + BLOCK_2MIB; + return result; +} diff --git a/src/core/guest.h b/src/core/guest.h index 9df66e4b..ea0e6434 100644 --- a/src/core/guest.h +++ b/src/core/guest.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include #include @@ -264,7 +265,11 @@ typedef struct { uint64_t offset; /* File offset (for /proc/self/maps display) */ int backing_fd; /* Duplicated host fd for file-backed mappings, or -1 */ bool shared; /* MAP_SHARED (writes should propagate) */ - bool noreserve; /* MAP_NORESERVE: PTEs deferred until fault */ + bool noreserve; /* Lazy region: PTEs and zeroing deferred until first + * touch. Set for MAP_NORESERVE and for all private + * anonymous mappings (sys_mmap folds the latter into + * the tracked MAP_NORESERVE bit; not guest visible). + */ bool backing_ro; /* MAP_SHARED region whose backing_fd was opened * without write access, so its Linux max_prot is * capped to PROT_READ. sys_mprotect must reject any @@ -301,6 +306,8 @@ typedef struct { * TLBI_BROADCAST -> X8 = 1 (TLBI VMALLE1IS, broadest) * TLBI_RANGE -> X8 = 3, X9 = start VA, X10 = page count * (TLBI VAE1IS loop preserves unrelated TLB entries) + * TLBI_RANGE_LARGE -> X8 = 4, X9 = encoded range operand + * (single TLBI RVAE1IS) * X8 = 2 is reserved for the execve drop-frame marker the shim handles * separately; it is never produced by the accumulator. */ @@ -323,14 +330,12 @@ typedef enum { */ #define TLBI_SELECTIVE_MAX_PAGES 16 -/* Cap single-shot TLBI RVAE1IS at this many 4 KiB pages. With SCALE=0 the - * RVAE1IS operand encoding covers (NUM+1)*2 pages with NUM in [0..31], so a - * single instruction reaches 64 pages == 256 KiB. Beyond that the host would - * need SCALE=1 (NUM*64 step), which over-invalidates for the typical - * dynamic-linker RELRO / glibc-bring-up storm sizes seen in practice; stay at - * SCALE=0 for now and broadcast above 64 pages. +/* Cap single-shot TLBI RVAE1IS at this many 4 KiB pages. SCALE=0 covers up to + * 64 pages, SCALE=1 up to 2048 pages, and SCALE=2 much larger ranges. 32768 + * pages (128 MiB) covers the lazy fault-around window with one instruction + * while still fitting tlbi_request_t.pages in uint16_t. */ -#define TLBI_RVAE_MAX_PAGES 64 +#define TLBI_RVAE_MAX_PAGES 32768 /* TLBI RVAE1IS operand bit-field constants. Per ARM ARM DDI 0487J.a D8.7.6 the * operand layout is: @@ -346,28 +351,37 @@ typedef enum { */ #define RVAE_OPERAND_BADDR_MASK ((1ULL << 37) - 1) #define RVAE_OPERAND_NUM_SHIFT 39 +#define RVAE_OPERAND_SCALE_SHIFT 44 #define RVAE_OPERAND_TG_4KB (1ULL << 46) /* Pure encoder: build the TLBI RVAE1IS Xt operand from a 4 KiB-aligned VA and a - * page count in the SCALE=0 range (1..TLBI_RVAE_MAX_PAGES). Lives in the header - * as static inline so tlbi_request_emit_to_vcpu and any future caller - * (host-side unit tests included) compile to the same expression. NUM = - * ceil(pages / 2) - 1 over-invalidates odd page counts by exactly one page, - * which is a perf-only side effect (the extra invalidation evicts a neighbour - * TLB entry that the guest's next access reloads). pages < 2 is clamped to 2 - * because SCALE=0 NUM=0 means 2 pages -- the encoder cannot represent a single - * page through RVAE1IS; single-page callers go through the per-page VAE1IS path - * instead, but the clamp keeps the encoder total in any pathological input. + * page count in the supported SCALE=0..2 range. Lives in the header so the emit + * path and host unit tests use the same expression. Each NUM step covers + * 2^(5*SCALE+1) pages; the normalizer aligns and widens callers to that unit. + * pages < 2 is clamped to 2 because SCALE=0 NUM=0 is the smallest encodable + * range. Single-page callers normally use VAE1IS instead. */ +static inline uint64_t tlbi_rvae_unit_pages(uint64_t pages) +{ + if (pages <= 64) + return 2; /* SCALE=0 */ + if (pages <= 2048) + return 64; /* SCALE=1 */ + return 2048; /* SCALE=2 */ +} + static inline uint64_t tlbi_rvae1is_operand(uint64_t start_va, uint16_t pages) { if (pages < 2) pages = 2; + uint64_t scale = pages <= 64 ? 0 : (pages <= 2048 ? 1 : 2); + uint64_t unit_pages = 1ULL << (5 * scale + 1); uint64_t baddr = (start_va >> 12) & RVAE_OPERAND_BADDR_MASK; - uint64_t num = ((pages + 1) / 2) - 1; + uint64_t num = ((pages + unit_pages - 1) / unit_pages) - 1; if (num > 31) num = 31; - return baddr | (num << RVAE_OPERAND_NUM_SHIFT) | RVAE_OPERAND_TG_4KB; + return baddr | (num << RVAE_OPERAND_NUM_SHIFT) | + (scale << RVAE_OPERAND_SCALE_SHIFT) | RVAE_OPERAND_TG_4KB; } /* Runtime feature flag: TRUE when the host PE implements FEAT_TLBIRANGE @@ -383,8 +397,8 @@ typedef struct { * after the TLBI sequence. 0 = data-only * change, skip the I-cache invalidation. */ - uint16_t pages; /* Page count when kind == TLBI_RANGE (1..MAX) */ - uint64_t start; /* Page-aligned VA when kind == TLBI_RANGE */ + uint16_t pages; /* Page count for either range kind (1..MAX) */ + uint64_t start; /* Page-aligned VA for either range kind */ } tlbi_request_t; /* Layout contract: 16 bytes (1+1+2+4 padding+8). Documents the padding and pins @@ -437,6 +451,44 @@ typedef struct { uint64_t next; /* Bump offset; (next + BLOCK_2MIB) > size means full */ } guest_overflow_t; +/* One conservative "may contain nonzero bytes" bit per 2 MiB primary-slab + * block. The largest supported slab is 1 TiB, so this costs 64 KiB per guest. + * All access is serialized by mmap_lock. + */ +#define GUEST_DIRTY_BLOCKS_MAX ((1ULL << 40) / BLOCK_2MIB) +#define GUEST_DIRTY_WORDS (GUEST_DIRTY_BLOCKS_MAX / 64) + +/* Host-side occupancy index for low-VA TTBR0 mappings. One bit per 2 MiB block + * records whether that block contains at least one valid L2/L3 PTE; a + * second-level bitmap records which occupancy words are non-zero. This lets + * huge lazy mmap/munmap ranges skip untouched address space without walking one + * page-table slot per GiB. The index is separate from dirty_blocks: a read-only + * mapping can have valid PTEs without dirty backing bytes. + */ +#define GUEST_PTE_PRESENT_BLOCKS_MAX GUEST_DIRTY_BLOCKS_MAX +#define GUEST_PTE_PRESENT_WORDS (GUEST_PTE_PRESENT_BLOCKS_MAX / 64) +#define GUEST_PTE_PRESENT_SUMMARY_WORDS (GUEST_PTE_PRESENT_WORDS / 64) +#define GUEST_PTE_PRESENT_LIMIT (GUEST_PTE_PRESENT_BLOCKS_MAX * BLOCK_2MIB) + +enum { + GUEST_MATERIALIZE_CLEAN_SKIP = 0, + GUEST_MATERIALIZE_DIRTY_MEMSET, + GUEST_MATERIALIZE_ALREADY_VALID, + GUEST_MATERIALIZE_WINDOW_BYTES, + GUEST_MATERIALIZE_STATS_N, +}; + +/* Dirty-block zeroing claims. A claim makes its block's invalid PTE window + * stable while the expensive host memset runs without mmap_lock. Waiters use + * one guest-wide condition variable; the fixed table bounds host allocation and + * is ample for the vCPU limit. + */ +#define GUEST_MATERIALIZE_CLAIMS 64 +typedef struct { + uint64_t start, end; + bool active; +} guest_materialize_claim_t; + /* Guest state. */ typedef struct { void *host_base; /* Host pointer to allocated guest memory */ @@ -564,6 +616,13 @@ typedef struct { */ _Atomic uint64_t pt_gen; + uint64_t pte_present_blocks[GUEST_PTE_PRESENT_WORDS]; + uint64_t pte_present_summary[GUEST_PTE_PRESENT_SUMMARY_WORDS]; + uint64_t dirty_blocks[GUEST_DIRTY_WORDS]; + uint64_t materialize_stats[GUEST_MATERIALIZE_STATS_N]; + guest_materialize_claim_t materialize_claims[GUEST_MATERIALIZE_CLAIMS]; + pthread_cond_t materialize_cond; + /* Optional HVC 6 embedder extension hook. * * Native AArch64 guests reach this through HVC 6. When the build enables @@ -685,8 +744,8 @@ static inline void tlbi_request_emit_to_vcpu(hv_vcpu_t vcpu) hv_vcpu_set_reg(vcpu, HV_REG_X11, cpu_tlbi_req.icache_flush ? 1 : 0); break; case TLBI_RANGE_LARGE: { - /* Single-shot TLBI RVAE1IS for ranges in (16..64] pages. The operand - * format and the SCALE=0 / TG=01 / ASID=0 assumptions are documented at + /* Single-shot TLBI RVAE1IS for ranges above the selective cap. The + * SCALE/NUM format and TG=01 assumption are documented at * tlbi_rvae1is_operand above. ASID stays 0 because the shim runs * single-ASID (TCR_EL1.A1=0, TTBR0 ASID=0; rosetta does not allocate a * separate ASID). If a future change introduces non-zero ASIDs, the @@ -708,6 +767,31 @@ static inline void tlbi_request_emit_to_vcpu(hv_vcpu_t vcpu) tlbi_request_clear(); } +/* RVAE1IS requires BaseADDR alignment to its SCALE granule. Widen a requested + * interval to that granule, repeating if widening crosses a SCALE threshold. + * Over-invalidation is architecturally harmless and preserves unrelated TLB + * entries far better than a VMALLE1IS broadcast. + */ +static inline bool tlbi_rvae_normalize(uint64_t *start, uint64_t *end) +{ + for (int pass = 0; pass < 3; pass++) { + uint64_t pages = (*end - *start) >> 12; + if (pages <= TLBI_SELECTIVE_MAX_PAGES) + return true; + uint64_t unit_pages = tlbi_rvae_unit_pages(pages); + uint64_t unit_bytes = unit_pages << 12; + uint64_t s = *start & ~(unit_bytes - 1); + if (*end > UINT64_MAX - (unit_bytes - 1)) + return false; + uint64_t e = (*end + unit_bytes - 1) & ~(unit_bytes - 1); + *start = s; + *end = e; + if (((e - s) >> 12) <= unit_pages * 32) + return ((e - s) >> 12) <= TLBI_RVAE_MAX_PAGES; + } + return false; +} + static inline void tlbi_request_range(uint64_t start, uint64_t end) { if (cpu_tlbi_req.kind == TLBI_BROADCAST) @@ -737,6 +821,13 @@ static inline void tlbi_request_range(uint64_t start, uint64_t end) */ uint64_t large_cap = g_tlbi_range_supported ? TLBI_RVAE_MAX_PAGES : TLBI_SELECTIVE_MAX_PAGES; + if (g_tlbi_range_supported && n > TLBI_SELECTIVE_MAX_PAGES) { + if (!tlbi_rvae_normalize(&s, &e)) { + tlbi_request_broadcast(); + return; + } + n = (e - s) >> 12; + } if (n > large_cap) { tlbi_request_broadcast(); return; @@ -768,6 +859,13 @@ static inline void tlbi_request_range(uint64_t start, uint64_t end) uint64_t us = s < es ? s : es; uint64_t ue = e > ee ? e : ee; uint64_t un = (ue - us) >> 12; + if (g_tlbi_range_supported && un > TLBI_SELECTIVE_MAX_PAGES) { + if (!tlbi_rvae_normalize(&us, &ue)) { + tlbi_request_broadcast(); + return; + } + un = (ue - us) >> 12; + } if (un > large_cap) { tlbi_request_broadcast(); return; @@ -1006,11 +1104,22 @@ int guest_install_va_pages(guest_t *g, uint64_t gpa, int perms); -/* Query whether a 2 MiB TTBR0 VA block already has a leaf mapping. - * Returns true only for a present L2 block descriptor. +/* Query whether a 2 MiB TTBR0 VA block already has any L2 entry, either a block + * descriptor or an L3 table descriptor. */ bool guest_va_block_mapped(const guest_t *g, uint64_t va); +/* Rebuild the low-VA PTE occupancy index from TTBR0. Used after fork restores + * page-table pages into a freshly initialized guest_t. + */ +void guest_rebuild_pte_present(guest_t *g); + +/* Reconcile the host-only 2 MiB occupancy index after EL1 has invalidated + * descriptors in [start,end). This observes PTEs only; it never writes a + * descriptor or requests another TLBI. Caller holds mmap_lock. + */ +void guest_retire_ptes_committed(guest_t *g, uint64_t start, uint64_t end); + /* Returns true when the VA range [va, va+size) overlaps the user-VA kbuf alias * window [KBUF_USER_VA, KBUF_USER_VA+KBUF_SIZE). Callers that install TTBR0 * mappings (the future rosetta_finalize, sys_mmap MAP_FIXED touching this @@ -1033,6 +1142,39 @@ static inline bool guest_kbuf_user_va_overlap(uint64_t va, uint64_t size) */ void *guest_ptr(const guest_t *g, uint64_t gva); +/* Like guest_ptr_avail but never triggers lazy fault-in. For callers that + * already hold mmap_lock or must avoid acquiring it. + */ +void *guest_ptr_avail_nofault(const guest_t *g, + uint64_t gva, + uint64_t *avail, + int required_perms); + +/* Materialize any lazy (deferred-PTE) blocks intersecting [gva, gva+len) so + * later resolves under locks that rank after mmap_lock (futex buckets) do not + * have to. Takes mmap_lock; call only from lock-free context. + * + * Returns 0 if anything was (or already is) materialized, -1 otherwise; callers + * that merely pre-fault can ignore the result. + */ +int guest_lazy_faultin(const guest_t *g, uint64_t gva, uint64_t len); + +/* Same as guest_lazy_faultin for callers that already hold mmap_lock (e.g. + * SC_LOCKED syscall handlers about to guest_read/guest_write a lazy range: the + * resolve-time hook would self-deadlock re-acquiring mmap_lock, so they must + * materialize up front through this variant). + */ +int guest_lazy_faultin_locked(const guest_t *g, uint64_t gva, uint64_t len); + +/* Smallest 2MiB-block-aligned va' in [va, end) containing at least one valid + * TTBR0 PTE, or end if none. Low VA uses the host-side hierarchical occupancy + * bitmap; non-identity/high VA falls back to the page-table hierarchy. Locking: + * callers MUST hold mmap_lock. + */ +uint64_t guest_va_next_present_block(const guest_t *g, + uint64_t va, + uint64_t end); + /* Get a host pointer for a guest virtual address (write access). * Returns NULL if gva is out of bounds or not writable. */ @@ -1092,6 +1234,12 @@ size_t guest_write_partial(guest_t *g, const void *src, size_t len); +/* Same copy without lazy materialization. Callers that already hold mmap_lock + * can use this after guest_lazy_faultin_locked(); an invalid destination then + * fails instead of recursively trying to acquire mmap_lock. + */ +int guest_write_nofault(guest_t *g, uint64_t gva, const void *src, size_t len); + /* Optimized host-to-guest copy for small fixed-size outputs. Uses a direct * guest pointer when the full range is contiguous and writable, otherwise falls * back to guest_write() for boundary-crossing safety. @@ -1366,12 +1514,33 @@ bool guest_region_range_has_ro_shared_backing(const guest_t *g, uint64_t start, uint64_t end); -/* Try to materialize a lazy (MAP_NORESERVE) page at the given offset. Called - * from the data/instruction abort handler when the faulting address falls - * within a noreserve region. Creates page table entries for one 2MiB block - * containing the fault address, zeros the memory, and clears the noreserve flag - * for the materialized sub-range. +/* Try to materialize a lazy (deferred-PTE: private anonymous or MAP_NORESERVE) + * page at the given offset. Called from the data/instruction abort handler when + * the faulting address falls within a lazy region, and from the host-access + * fault-in path when a syscall targets a lazy range the guest has not touched + * yet. Creates page table entries for one 2MiB block containing the fault + * address. A slab block known to be clean skips zeroing; a possibly-dirty block + * zeros invalid pages before publishing them. A block that is already valid + * returns success without re-zeroing (concurrent-fault idempotence). * Returns 0 on success (caller should TLBI and retry), -1 if the offset is not - * in a noreserve region. + * in a lazy region or the region is PROT_NONE. Locking: callers MUST hold + * mmap_lock. */ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset); + +/* Guest-fault variant with per-vCPU sequential fault-around. */ +int guest_materialize_lazy_fault(guest_t *g, uint64_t fault_offset); + +/* Dirty-map and in-flight-claim helpers. Callers hold mmap_lock. */ +bool guest_block_may_be_dirty(const guest_t *g, uint64_t block_start); +void guest_dirty_mark_range(guest_t *g, uint64_t start, uint64_t end); +void guest_dirty_clear_zeroed_range(guest_t *g, uint64_t start, uint64_t end); +void guest_materialize_wait_range_locked(guest_t *g, + uint64_t start, + uint64_t end); +void guest_materialize_wait_all_locked(guest_t *g); + +/* Whether the 4KiB page containing va has a valid stage-1 descriptor. Locking: + * callers MUST hold mmap_lock. + */ +bool guest_va_pte_valid(guest_t *g, uint64_t va); diff --git a/src/core/launch.c b/src/core/launch.c index bbe068b7..5b8a195b 100644 --- a/src/core/launch.c +++ b/src/core/launch.c @@ -26,6 +26,7 @@ #include "core/bootstrap.h" #include "core/guest.h" +#include "core/mmap-fastpath.h" #include "core/shim-globals.h" #include "core/sysroot.h" @@ -192,6 +193,14 @@ int elfuse_launch(const launch_args_t *args) 0) goto fail; + /* Tracing/debuggers require every mmap to reach host dispatch so syscall + * observation and region state advance in lockstep. This also prevents a + * trace-gated shim from leaving invisible arena reservations that perturb + * the host slow path's address-hint behavior. + */ + if (args->verbose || args->gdb_port > 0) + mmap_fastpath_disable(&g); + /* GDB setup must happen before the first run so entry-stop and hardware * breakpoints can affect the initial vCPU. */ diff --git a/src/core/mmap-fastpath.h b/src/core/mmap-fastpath.h new file mode 100644 index 00000000..6ca06e57 --- /dev/null +++ b/src/core/mmap-fastpath.h @@ -0,0 +1,204 @@ +/* + * Per-vCPU EL1 anonymous-mmap consumer rings. + * + * The host produces arenas, consumes mmap/munmap publications, and returns + * metadata-committed holes through a reverse SPSC ring. EL1 first-fit consumes + * those private extents before bump-allocating fresh VA. Control blocks live in + * the EL1-only shim-data mapping and are selected from SP_EL1's per-thread + * stack slot, so no guest-visible register ABI is consumed. + */ + +#pragma once + +#include +#include +#include +#include + +#include "core/guest.h" +#include "proved/mmap-fastpath.h" + +typedef struct thread_entry thread_entry_t; + +#define SHIM_MMAP_CONTROL_BASE 0x20000u +#define SHIM_MMAP_CONTROL_STRIDE 0x1000u +#define SHIM_MMAP_RING_SIZE 32u +#define SHIM_MMAP_CTRL_ENABLED 0x1u +#define SHIM_MMAP_CTRL_TLBIRANGE 0x2u + +/* Host page-table writers set this gate before changing an arena descriptor or + * a stage-1 PTE. Each EL1 producer announces itself in its private control + * before rechecking the gate. This is a writer-vs-per-vCPU-reader handshake, + * not an allocator lock: fast munmap producers never write a shared cache line + * or wait for one another. + */ +#define SHIM_MMAP_PT_GATE_OFF 0x1160u + +#define SHIM_MUNMAP_RETIRE_RING_SIZE 32u +#define SHIM_MUNMAP_RETIRE_OFF 0x400u +#define SHIM_MUNMAP_RETIRE_BYTES_SOFT (256ULL * 1024 * 1024) +#define SHIM_MUNMAP_RETIRE_F_ARENA_SLOT_MASK 0x3fu +#define SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT 6u +#define SHIM_MUNMAP_RETIRE_F_CHARGE_MASK 0xffffffc0u + +/* MMAP_FAST_ARENA_MIN/MAX/TARGET_ENTRIES and MMAP_FAST_PUBLICATION_WINDOW are + * defined in proved/mmap-fastpath.h, included above: that header's sizing + * arithmetic is proved against those exact values, so this is the one + * definition rather than a copy a proof cannot see drift from. + * + * MMAP_FAST_PUBLICATION_WINDOW must stay a power of two: the sequence counter + * indexes the window by masking. + */ + +enum { + SHIM_MMAP_COUNTER_SHAPE_MISS = 0, + SHIM_MMAP_COUNTER_CAPACITY_MISS, + SHIM_MMAP_COUNTER_RING_FULL, + SHIM_MMAP_COUNTER_GENERATION_STALE, + SHIM_MMAP_COUNTER_ATTENTION, + SHIM_MMAP_COUNTER_HIT, + SHIM_MMAP_COUNTERS_N, +}; + +typedef struct { + uint64_t addr; + uint64_t len; + uint64_t prot; +} shim_mmap_entry_t; + +typedef struct munmap_retire_entry { + uint64_t addr; + uint64_t length; + uint32_t arena_generation; + uint32_t flags; +} munmap_retire_entry_t; + +typedef struct munmap_retire_ring { + _Atomic uint32_t head; /* host consumer */ + _Atomic uint32_t tail; /* EL1 producer */ + /* Monotonic accounting of conservative materialized-byte upper bounds. */ + _Atomic uint64_t produced_bytes; /* EL1 producer */ + _Atomic uint64_t consumed_bytes; /* host consumer */ + _Atomic uint32_t producer_active; + /* Advisory; never forces the producer to exit. */ + _Atomic uint32_t cleanup_requested; + munmap_retire_entry_t entries[SHIM_MUNMAP_RETIRE_RING_SIZE]; +} munmap_retire_ring_t; + + +typedef struct { + _Atomic uint32_t generation; /* host publish word */ + _Atomic uint32_t consumer_generation; /* EL1 generation ack */ + _Atomic uint32_t flags; /* host-owned enable bits */ + _Atomic uint32_t head; /* host consumer cursor */ + _Atomic uint32_t tail; /* EL1 producer cursor */ + uint32_t _pad0; + _Atomic uint64_t arena_base; + _Atomic uint64_t arena_limit; + _Atomic uint64_t cursor; /* EL1 bump cursor */ + uint64_t next_arena_size; /* most recently selected generation size */ + uint64_t publication_seq; /* host-only: registrations, rotates the window */ + shim_mmap_entry_t ring[SHIM_MMAP_RING_SIZE]; + _Atomic uint64_t counters[SHIM_MMAP_COUNTERS_N]; + uint64_t refill_count; + uint64_t recycle_count; + uint64_t peak_arena_size; + _Atomic uint32_t materialized_generation; + uint32_t _pad1; + _Atomic uint64_t materialized_start; + _Atomic uint64_t materialized_end; + uint8_t _pad2[SHIM_MUNMAP_RETIRE_OFF - 0x3a0]; + munmap_retire_ring_t retire; + + /* Host-only refill policy history; see MMAP_FAST_PUBLICATION_WINDOW. Kept + * past the rings so adding it moves no offset the EL1 side depends on. + */ + uint64_t publication_window[MMAP_FAST_PUBLICATION_WINDOW]; + + /* EL1-incremented, host-read: how many munmap() calls found the retirement + * ring near full (see el1_munmap's near-full check) and fell back to a + * synchronous host trap instead of publishing into the ring. Placed past + * every fixed EL1 ABI offset, like publication_window above. + */ + _Atomic uint64_t munmap_retire_near_full; + + /* EL1 sets this after publishing mmap or munmap work. The host uses it only + * to avoid an empty opportunistic VM-exit drain; lock-taking paths still + * drain unconditionally. + */ + _Atomic uint32_t pending_work; +} shim_mmap_control_t; + +_Static_assert(offsetof(shim_mmap_control_t, retire) == SHIM_MUNMAP_RETIRE_OFF, + "EL1 retire-ring offset ABI"); +_Static_assert(sizeof(shim_mmap_control_t) <= SHIM_MMAP_CONTROL_STRIDE, + "mmap control exceeds per-vCPU stride"); + +/* Provision the main vCPU before guest entry. Worker vCPUs provision lazily on + * their first eligible mmap so short-lived threads do not allocate an unused + * arena. + */ +void mmap_fastpath_prepare_vcpu(guest_t *g, thread_entry_t *t); + +/* Drain every per-vCPU SPSC ring. Caller holds mmap_lock. */ +void mmap_fastpath_drain_locked(guest_t *g); + +/* Opportunistically drain publications and retirements at a natural VM exit. + * Safe before every exit handler; it acquires mmap_lock internally. When a + * fork-family syscall is pending, skip the arena top-up that the syscall will + * immediately revoke. + */ +void mmap_fastpath_drain_vmexit(guest_t *g, bool fork_family_pending); + +/* True when the stopped current vCPU was interrupted in the middle of its EL1 + * producer critical section. A cancellation exit must resume it before host + * drain can close the PT gate. + */ +bool mmap_fastpath_current_producer_active(const guest_t *g); + +/* Mark every arena intersecting a successfully materialized lazy range. Caller + * holds mmap_lock. EL1 may skip its PTE walk only while this marker differs + * from the arena's current generation. + */ +void mmap_fastpath_note_materialized_locked(guest_t *g, + uint64_t start, + uint64_t end); + +/* Refill the current vCPU after an eligible mmap slow-path. request_len is + * page-rounded; requests above MMAP_FAST_ARENA_MAX leave the arena untouched. + * Caller holds mmap_lock. + */ +void mmap_fastpath_refill_current_locked(guest_t *g, uint64_t request_len); + +/* Fulfil an eligible mmap that reached HVC (capacity/ring/generation fallback) + * directly from the current vCPU's refilled arena. Metadata is committed by the + * host immediately, so no mmap publication entry is needed. Caller holds + * mmap_lock. + */ +bool mmap_fastpath_allocate_current_locked(guest_t *g, + uint64_t request_len, + uint64_t *addr_out); +bool mmap_fastpath_allocate_current_publication_only(guest_t *g, + uint64_t request_len, + uint64_t *addr_out); + +/* Give an explicit slow-path hint precedence over this stopped vCPU's + * unconsumed arena tail. Caller holds mmap_lock. + */ +void mmap_fastpath_release_current_hint_locked(guest_t *g, + uint64_t addr, + uint64_t length); + +/* Revoke all arenas while sibling vCPUs are quiesced. Caller holds mmap_lock. + */ +void mmap_fastpath_revoke_all_locked(guest_t *g, bool shrink_high_water); + +/* Disable the feature before first guest entry (debugger/observability). */ +void mmap_fastpath_disable(guest_t *g); + +/* Advance *start past an EL1 arena reservation that overlaps length bytes. */ +void mmap_fastpath_skip_reserved(const guest_t *g, + uint64_t *start, + uint64_t length, + uint64_t align, + uint64_t max_addr); diff --git a/src/core/shim-globals.c b/src/core/shim-globals.c index 517226ad..3dcbd33d 100644 --- a/src/core/shim-globals.c +++ b/src/core/shim-globals.c @@ -10,6 +10,7 @@ */ #include +#include #include #include #include @@ -19,6 +20,7 @@ #include "hvutil.h" #include "core/guest.h" +#include "core/mmap-fastpath.h" #include "core/shim-globals.h" #include "core/vdso.h" #include "debug/log.h" @@ -97,6 +99,84 @@ _Static_assert(SHIM_GLOBALS_SIZE <= BLOCK_2MIB, _Static_assert(SHIM_COUNTERS_OFF + SHIM_COUNTERS_N * 8 <= SHIM_IDENTITY_OFF_PGID, "counter array must not overlap the PGID slot"); +_Static_assert(SHIM_MMAP_CONTROL_BASE == 0x20000, + "EL1 mmap fast path hard-codes control base 0x20000"); +_Static_assert(SHIM_MMAP_CONTROL_STRIDE == 0x1000, + "EL1 mmap fast path hard-codes control stride 0x1000"); +_Static_assert(SHIM_MMAP_RING_SIZE == 32, + "EL1 mmap fast path hard-codes 32 ring entries"); +_Static_assert(offsetof(shim_mmap_control_t, generation) == 0, + "EL1 mmap generation offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, consumer_generation) == 4, + "EL1 mmap consumer-generation offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, flags) == 8, + "EL1 mmap flags offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, head) == 12, + "EL1 mmap head offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, tail) == 16, + "EL1 mmap tail offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, arena_base) == 24, + "EL1 mmap arena-base offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, arena_limit) == 32, + "EL1 mmap arena-limit offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, cursor) == 40, + "EL1 mmap cursor offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, next_arena_size) == 48, + "mmap next-arena-size offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, publication_seq) == 56, + "mmap publication-seq offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, ring) == 64, + "EL1 mmap ring offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, counters) == 0x340, + "EL1 mmap counter offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, materialized_generation) == 0x388, + "EL1 mmap materialized-generation offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, materialized_start) == 0x390 && + offsetof(shim_mmap_control_t, materialized_end) == 0x398, + "EL1 mmap materialized bounds offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, retire) == 0x400, + "EL1 munmap retire offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, produced_bytes) == 8, + "EL1 munmap produced-byte offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, consumed_bytes) == 16, + "EL1 munmap consumed-byte offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, producer_active) == 24, + "EL1 munmap active offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, cleanup_requested) == 28, + "EL1 munmap cleanup-request offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, entries) == 32, + "EL1 munmap entries offset drift"); +_Static_assert(SHIM_MUNMAP_RETIRE_RING_SIZE == 32 && MAX_THREADS == 64, + "EL1 munmap ring/arena scan constants drift"); +_Static_assert(SHIM_MUNMAP_RETIRE_BYTES_SOFT == 0x10000000ULL, + "EL1 munmap byte advisory threshold drift"); +_Static_assert(SHIM_MUNMAP_RETIRE_F_ARENA_SLOT_MASK == 0x3f && + SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT == 6, + "EL1 munmap retire flag encoding drift"); + +/* Host-only, so no EL1 offset depends on it; the assert pins it past every ring + * instead, which is what keeps adding it from moving the shared layout. + */ +_Static_assert(offsetof(shim_mmap_control_t, publication_window) > + offsetof(shim_mmap_control_t, retire), + "publication window must not precede the EL1-visible rings"); +_Static_assert((MMAP_FAST_PUBLICATION_WINDOW & + (MMAP_FAST_PUBLICATION_WINDOW - 1)) == 0, + "publication window size must be a power of two"); +_Static_assert((MMAP_FAST_ARENA_MAX >> 12) <= + (SHIM_MUNMAP_RETIRE_F_CHARGE_MASK >> + SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT), + "munmap retire flags cannot encode maximum arena charge"); +_Static_assert(SHIM_MMAP_PT_GATE_OFF >= SHIM_GLOBALS_SIZE && + SHIM_MMAP_PT_GATE_OFF + sizeof(uint32_t) <= + SHIM_MMAP_CONTROL_BASE, + "host PT gate overlaps shim globals or mmap controls"); +_Static_assert(sizeof(shim_mmap_control_t) <= SHIM_MMAP_CONTROL_STRIDE, + "per-vCPU mmap control exceeds its shim-data stride"); +_Static_assert(SHIM_MMAP_CONTROL_BASE + + MAX_THREADS * SHIM_MMAP_CONTROL_STRIDE <= + BLOCK_2MIB - MAX_THREADS * 4096, + "mmap controls overlap per-vCPU EL1 stacks"); static uint8_t *cache_base(const guest_t *g) { @@ -127,7 +207,13 @@ static void urandom_ring_unlock(_Atomic uint32_t *lock_p) void shim_globals_init(guest_t *g) { - memset(cache_base(g), 0, SHIM_GLOBALS_SIZE); + /* mmap controls occupy a separate low shim-data range. Init/exec/fork child + * all call this while no sibling can execute, so clearing the whole control + * array also prevents a recycled SP_EL1 slot from inheriting an arena + * published to its previous owner. + */ + memset(cache_base(g), 0, + SHIM_MMAP_CONTROL_BASE + MAX_THREADS * SHIM_MMAP_CONTROL_STRIDE); } void shim_globals_publish_pid(guest_t *g, int64_t pid, int64_t ppid) @@ -446,12 +532,10 @@ static const char *const counter_names[SHIM_COUNTERS_N] = { [SHIM_COUNTER_URANDOM_HIT] = "URANDOM_HIT", [SHIM_COUNTER_GETRANDOM_HIT] = "GETRANDOM_HIT", [SHIM_COUNTER_PGSID_HIT] = "PGSID_HIT", - - /* Slots 12..15 (SHIM_COUNTERS_N == 16) are intentionally unnamed; the dump - * prints "(reserved)" so they appear in the output when non-zero, which - * would flag an out-of-band increment. Bind a name here when a future EL1 - * service claims one of these slots. - */ + [SHIM_COUNTER_FAULT_MATERIALIZE] = "FAULT_MATERIALIZE", + [SHIM_COUNTER_FAULT_TLBI_VAE] = "FAULT_TLBI_VAE", + [SHIM_COUNTER_FAULT_TLBI_RVAE] = "FAULT_TLBI_RVAE", + [SHIM_COUNTER_FAULT_TLBI_BCAST] = "FAULT_TLBI_BCAST", }; uint64_t shim_globals_counter_get(const guest_t *g, unsigned slot) @@ -464,6 +548,16 @@ uint64_t shim_globals_counter_get(const guest_t *g, unsigned slot) return atomic_load_explicit(slot_p, memory_order_relaxed); } +void shim_globals_counter_inc(guest_t *g, unsigned slot) +{ + if (!shim_globals_stats_enabled() || slot >= SHIM_COUNTERS_N) + return; + uint8_t *page = (uint8_t *) g->host_base + g->shim_data_base; + _Atomic uint64_t *slot_p = + (_Atomic uint64_t *) (page + SHIM_COUNTERS_OFF) + slot; + atomic_fetch_add_explicit(slot_p, 1, memory_order_relaxed); +} + void shim_globals_counters_dump(const guest_t *g) { fprintf(stderr, "shim-stats (pid=%lld)\n", (long long) proc_get_pid()); @@ -475,6 +569,67 @@ void shim_globals_counters_dump(const guest_t *g) fprintf(stderr, " %-20s %llu\n", name ? name : "(reserved)", (unsigned long long) v); } + + static const char *const mmap_counter_names[SHIM_MMAP_COUNTERS_N] = { + [SHIM_MMAP_COUNTER_SHAPE_MISS] = "MMAP_SHAPE_MISS", + [SHIM_MMAP_COUNTER_CAPACITY_MISS] = "MMAP_CAPACITY_MISS", + [SHIM_MMAP_COUNTER_RING_FULL] = "MMAP_RING_FULL", + [SHIM_MMAP_COUNTER_GENERATION_STALE] = "MMAP_GENERATION_STALE", + [SHIM_MMAP_COUNTER_ATTENTION] = "MMAP_ATTENTION", + [SHIM_MMAP_COUNTER_HIT] = "MMAP_HIT", + }; + uint64_t mmap_counters[SHIM_MMAP_COUNTERS_N] = {0}; + uint64_t refill_count = 0, recycle_count = 0; + uint64_t current_max = 0, peak_max = 0; + uint64_t munmap_retire_near_full = 0; + const uint8_t *shim_data = + (const uint8_t *) g->host_base + g->shim_data_base; + for (int slot = 0; slot < MAX_THREADS; slot++) { + const shim_mmap_control_t *c = + (const shim_mmap_control_t *) (shim_data + SHIM_MMAP_CONTROL_BASE + + (uint64_t) slot * + SHIM_MMAP_CONTROL_STRIDE); + for (unsigned i = 0; i < SHIM_MMAP_COUNTERS_N; i++) + mmap_counters[i] += + atomic_load_explicit(&c->counters[i], memory_order_relaxed); + refill_count += c->refill_count; + recycle_count += c->recycle_count; + if (c->next_arena_size > current_max) + current_max = c->next_arena_size; + if (c->peak_arena_size > peak_max) + peak_max = c->peak_arena_size; + munmap_retire_near_full += atomic_load_explicit( + &c->munmap_retire_near_full, memory_order_relaxed); + } + for (unsigned i = 0; i < SHIM_MMAP_COUNTERS_N; i++) + fprintf(stderr, " %-20s %llu\n", mmap_counter_names[i], + (unsigned long long) mmap_counters[i]); + fprintf(stderr, " %-20s %llu\n", "MMAP_REFILL", + (unsigned long long) refill_count); + fprintf(stderr, " %-20s %llu\n", "MMAP_RECYCLE", + (unsigned long long) recycle_count); + fprintf(stderr, " %-20s %llu\n", "MUNMAP_RETIRE_NEAR_FULL", + (unsigned long long) munmap_retire_near_full); + fprintf(stderr, " %-20s %llu\n", "MMAP_ARENA_CURRENT", + (unsigned long long) current_max); + fprintf(stderr, " %-20s %llu\n", "MMAP_ARENA_PEAK", + (unsigned long long) peak_max); + uint64_t high_water = + g->mmap_next > MMAP_BASE ? g->mmap_next - MMAP_BASE : 0; + fprintf(stderr, " %-20s %llu\n", "MMAP_HIGH_WATER", + (unsigned long long) high_water); + fprintf(stderr, " %-20s %llu\n", "FAULT_CLEAN_SKIP", + (unsigned long long) + g->materialize_stats[GUEST_MATERIALIZE_CLEAN_SKIP]); + fprintf(stderr, " %-20s %llu\n", "FAULT_DIRTY_MEMSET", + (unsigned long long) + g->materialize_stats[GUEST_MATERIALIZE_DIRTY_MEMSET]); + fprintf(stderr, " %-20s %llu\n", "FAULT_ALREADY_VALID", + (unsigned long long) + g->materialize_stats[GUEST_MATERIALIZE_ALREADY_VALID]); + fprintf(stderr, " %-20s %llu\n", "FAULT_WINDOW_BYTES", + (unsigned long long) + g->materialize_stats[GUEST_MATERIALIZE_WINDOW_BYTES]); } static pthread_once_t stats_once = PTHREAD_ONCE_INIT; diff --git a/src/core/shim-globals.h b/src/core/shim-globals.h index 14cb24f1..049775d4 100644 --- a/src/core/shim-globals.h +++ b/src/core/shim-globals.h @@ -165,7 +165,7 @@ * not in urandom bitmap, len zero, len over inline cap, ring fill below * request, ring wrap, EL0 buffer probe failure). Slots 8..11 record fast-path * hits so bail rates can be computed against a hit denominator. Slots 12..15 - * are reserved. + * attribute lazy-fault materializations and the TLBI wire mode they emit. * * The shim hardcodes the byte offset of each slot; the static_asserts in * shim-globals.c keep the C-side macros and the assembly in sync. @@ -185,6 +185,10 @@ #define SHIM_COUNTER_URANDOM_HIT 9 #define SHIM_COUNTER_GETRANDOM_HIT 10 #define SHIM_COUNTER_PGSID_HIT 11 +#define SHIM_COUNTER_FAULT_MATERIALIZE 12 +#define SHIM_COUNTER_FAULT_TLBI_VAE 13 +#define SHIM_COUNTER_FAULT_TLBI_RVAE 14 +#define SHIM_COUNTER_FAULT_TLBI_BCAST 15 /* Extended identity slots: pgid and sid. * @@ -390,6 +394,7 @@ void shim_globals_refill_urandom_ring(guest_t *g); * when ELFUSE_SHIM_STATS is set. */ uint64_t shim_globals_counter_get(const guest_t *g, unsigned slot); +void shim_globals_counter_inc(guest_t *g, unsigned slot); void shim_globals_counters_dump(const guest_t *g); /* ELFUSE_SHIM_STATS env-var gate (idempotent / cached). When enabled the exit diff --git a/src/core/shim-mmap.c b/src/core/shim-mmap.c new file mode 100644 index 00000000..0b1c7196 --- /dev/null +++ b/src/core/shim-mmap.c @@ -0,0 +1,947 @@ +/* + * EL1 mmap-family syscall fast paths. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * This file is compiled freestanding and linked into the shim image. It may not + * call host code or a C runtime: the only state it consumes is the saved EL0 + * register frame, TPIDR_EL1's shim-data mapping, TTBR0_EL1, and the shared mmap + * control protocol. Keep architecture-only operations in the small helpers + * below; the synchronization and allocator policy remain ordinary C. + */ + +#include "core/shim-mmap.h" + +#include +#include +#include + +#include "core/mmap-fastpath.h" +#include "core/shim-globals.h" + +#define EL1_PAGE_SIZE 0x1000ULL +#define EL1_BLOCK_SIZE 0x200000ULL +#define EL1_WINDOW_1G 0x40000000ULL +#define EL1_SHIM_DATA_SIZE 0x200000ULL + +#define EL1_SYS_MUNMAP 215ULL +#define EL1_SYS_MMAP 222ULL + +#define EL1_PROT_READ 1ULL +#define EL1_PROT_WRITE 2ULL +#define EL1_PROT_RW (EL1_PROT_READ | EL1_PROT_WRITE) + +#define EL1_MAP_PRIVATE 0x02ULL +#define EL1_MAP_ANONYMOUS 0x20ULL +#define EL1_MAP_NORESERVE 0x4000ULL + +#define EL1_DESC_ADDR_MASK 0x0000fffffffff000ULL +#define EL1_DESC_TYPE_MASK 0x3ULL +#define EL1_DESC_BLOCK 0x1ULL +#define EL1_DESC_TABLE 0x3ULL + +#define EL1_L3_PTES_PER_BATCH 8u +#define EL1_TLBI_LEVEL_L2 2u +#define EL1_TLBI_LEVEL_L3 3u + +/* The bump path promises 2 MiB-aligned starts at or above this request size, so + * the host can back them with L2 blocks. + */ +#define EL1_ALIGN_THRESHOLD EL1_BLOCK_SIZE + +/* Arena descriptors are one per vCPU slot. shim-globals.c pins this against + * MAX_THREADS; the host thread header is not includable freestanding. + */ +#define EL1_ARENA_SLOTS 64u + +typedef struct { + uint8_t *shim_data; + shim_mmap_control_t *control; + _Atomic uint32_t *gate; + unsigned slot; +} el1_mmap_context_t; + +typedef enum { + EL1_GATE_ENTERED, + EL1_GATE_CLOSED_BEFORE_ANNOUNCE, + EL1_GATE_CLOSED_AFTER_ANNOUNCE, +} el1_gate_result_t; + +_Static_assert(EL1_SAVED_GPRS * sizeof(uint64_t) == 248, + "saved GPR frame layout changed"); +_Static_assert(sizeof(shim_mmap_entry_t) == 24, + "EL1 mmap publication ABI changed"); +_Static_assert(sizeof(munmap_retire_entry_t) == 24, + "EL1 retire publication ABI changed"); + +static inline uint64_t el1_read_tpidr(void) +{ + uint64_t value; + __asm__ volatile("mrs %0, tpidr_el1" : "=r"(value)); + return value; +} + +static inline uint64_t el1_read_ttbr0(void) +{ + uint64_t value; + __asm__ volatile("mrs %0, ttbr0_el1" : "=r"(value)); + return value; +} + +static inline void el1_dsb_ishst(void) +{ + __asm__ volatile("dsb ishst" ::: "memory"); +} + +static inline void el1_dsb_ish(void) +{ + __asm__ volatile("dsb ish" ::: "memory"); +} + +static inline void el1_isb(void) +{ + __asm__ volatile("isb" ::: "memory"); +} + +static inline void el1_tlbi_vale1is(uint64_t operand) +{ + __asm__ volatile("tlbi vale1is, %0" : : "r"(operand) : "memory"); +} + +static inline void el1_tlbi_rvale1is(uint64_t operand) +{ + __asm__ volatile("tlbi rvale1is, %0" : : "r"(operand) : "memory"); +} + +static inline void el1_tlbi_all(void) +{ + __asm__ volatile("tlbi vmalle1is" ::: "memory"); +} + +static inline uint64_t el1_desc_address(uint64_t descriptor) +{ + return descriptor & EL1_DESC_ADDR_MASK; +} + +static inline uint64_t el1_desc_type(uint64_t descriptor) +{ + return descriptor & EL1_DESC_TYPE_MASK; +} + +/* Guest VA is identity-mapped over the arena, so a descriptor's output address + * is also the VA of the next table. + */ +static inline _Atomic uint64_t *el1_table(uint64_t descriptor) +{ + return (_Atomic uint64_t *) (uintptr_t) el1_desc_address(descriptor); +} + +static inline uint64_t el1_block_start(uint64_t address) +{ + return address & ~(EL1_BLOCK_SIZE - 1); +} + +static inline uint64_t el1_next_block(uint64_t address) +{ + return el1_block_start(address) + EL1_BLOCK_SIZE; +} + +static inline uint64_t el1_min(uint64_t left, uint64_t right) +{ + return left < right ? left : right; +} + +static inline bool el1_add_overflow(uint64_t left, + uint64_t right, + uint64_t *result) +{ + return __builtin_add_overflow(left, right, result); +} + +static bool el1_page_round(uint64_t length, uint64_t *rounded) +{ + uint64_t value; + if (length == 0 || el1_add_overflow(length, EL1_PAGE_SIZE - 1, &value)) + return false; + *rounded = value & ~(EL1_PAGE_SIZE - 1); + return true; +} + +/* SP_EL1 stack tops are shim_data_end - slot*4KiB and controls are shim_data + + * 0x20000 + slot*4KiB, so the saved frame's page locates the control without + * consuming another system register. + */ +static el1_mmap_context_t el1_context(uint64_t *saved_gprs) +{ + uintptr_t shim_data = (uintptr_t) el1_read_tpidr(); + uintptr_t frame = (uintptr_t) saved_gprs; + uintptr_t stack_top = + (frame + EL1_PAGE_SIZE - 1) & ~(uintptr_t) (EL1_PAGE_SIZE - 1); + uintptr_t slot_bytes = shim_data + EL1_SHIM_DATA_SIZE - stack_top; + unsigned slot = (unsigned) (slot_bytes / EL1_PAGE_SIZE); + return (el1_mmap_context_t) { + .shim_data = (uint8_t *) shim_data, + .control = (shim_mmap_control_t *) (shim_data + SHIM_MMAP_CONTROL_BASE + + (uintptr_t) slot * + SHIM_MMAP_CONTROL_STRIDE), + .gate = (_Atomic uint32_t *) (shim_data + SHIM_MMAP_PT_GATE_OFF), + .slot = slot, + }; +} + +static inline bool el1_stats_enabled(const el1_mmap_context_t *context) +{ + return context->shim_data[SHIM_GLOBALS_OFF_STATS_EN] != 0; +} + +/* Each vCPU is the sole writer of its own counter slots. */ +static inline void el1_counter_increment(_Atomic uint64_t *counter) +{ + uint64_t value = atomic_load_explicit(counter, memory_order_relaxed); + atomic_store_explicit(counter, value + 1, memory_order_relaxed); +} + +static inline void el1_mmap_counter(const el1_mmap_context_t *context, + unsigned counter) +{ + if (el1_stats_enabled(context)) + el1_counter_increment(&context->control->counters[counter]); +} + +static inline void el1_attention_counter(const el1_mmap_context_t *context) +{ + if (!el1_stats_enabled(context)) + return; + _Atomic uint64_t *counter = + (_Atomic uint64_t *) (context->shim_data + SHIM_COUNTERS_OFF) + + SHIM_COUNTER_ATTN_BAIL; + el1_counter_increment(counter); +} + +static inline bool el1_attention_pending(const el1_mmap_context_t *context) +{ + return atomic_load_explicit((_Atomic uint32_t *) (context->shim_data + + SHIM_GLOBALS_OFF_ATTN), + memory_order_acquire) != 0; +} + +/* Join the host-writer handshake. Arena revoke and refill both run with the + * gate closed; announcing this per-vCPU producer prevents the host from + * invalidating a descriptor between the generation check and the publication. + * The second gate read closes the check-vs-announce race: if the host closed it + * in between, withdraw without touching a PTE. + */ +static el1_gate_result_t el1_gate_enter(el1_mmap_context_t *context) +{ + if (atomic_load_explicit(context->gate, memory_order_acquire) != 0) + return EL1_GATE_CLOSED_BEFORE_ANNOUNCE; + + atomic_store_explicit(&context->control->retire.producer_active, 1, + memory_order_release); + if (atomic_load_explicit(context->gate, memory_order_acquire) != 0) { + atomic_store_explicit(&context->control->retire.producer_active, 0, + memory_order_release); + return EL1_GATE_CLOSED_AFTER_ANNOUNCE; + } + return EL1_GATE_ENTERED; +} + +static inline void el1_gate_leave(el1_mmap_context_t *context) +{ + atomic_store_explicit(&context->control->retire.producer_active, 0, + memory_order_release); +} + +/* mmap: consume a host-prepared, per-vCPU VA arena for the exact anonymous RW + * shape used by allocators: + * + * mmap(NULL, len, PROT_READ|PROT_WRITE, + * MAP_PRIVATE|MAP_ANONYMOUS[|MAP_NORESERVE], fd, off) + * + * Each vCPU is the sole producer of its mmap publication ring and cursor. The + * host acquire-drains all publications whenever it takes mmap_lock. + */ + +static bool el1_mmap(el1_mmap_context_t *context, uint64_t *saved_gprs) +{ + shim_mmap_control_t *control = context->control; + + if (el1_attention_pending(context)) { + el1_mmap_counter(context, SHIM_MMAP_COUNTER_ATTENTION); + el1_attention_counter(context); + return false; + } + + uint64_t prot = saved_gprs[2]; + uint64_t flags = saved_gprs[3]; + uint64_t length; + if (saved_gprs[0] != 0 || prot != EL1_PROT_RW || + (flags & ~EL1_MAP_NORESERVE) != (EL1_MAP_PRIVATE | EL1_MAP_ANONYMOUS) || + !el1_page_round(saved_gprs[1], &length)) { + el1_mmap_counter(context, SHIM_MMAP_COUNTER_SHAPE_MISS); + return false; + } + + switch (el1_gate_enter(context)) { + case EL1_GATE_ENTERED: + break; + case EL1_GATE_CLOSED_BEFORE_ANNOUNCE: + case EL1_GATE_CLOSED_AFTER_ANNOUNCE: + el1_mmap_counter(context, SHIM_MMAP_COUNTER_CAPACITY_MISS); + return false; + } + + uint32_t generation = + atomic_load_explicit(&control->generation, memory_order_acquire); + if (generation != atomic_load_explicit(&control->consumer_generation, + memory_order_relaxed)) { + /* A revocation deliberately leaves the consumer generation stale. Ack + * only after the acquire load above, then make this syscall take HVC; + * the host will either leave the control disabled or publish a fresh + * arena. + */ + atomic_store_explicit(&control->consumer_generation, generation, + memory_order_relaxed); + el1_gate_leave(context); + el1_mmap_counter(context, SHIM_MMAP_COUNTER_GENERATION_STALE); + return false; + } + + if (!(atomic_load_explicit(&control->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) { + el1_gate_leave(context); + el1_mmap_counter(context, SHIM_MMAP_COUNTER_CAPACITY_MISS); + return false; + } + + /* Reserve publication capacity before mutating the bump cursor. */ + uint32_t head = atomic_load_explicit(&control->head, memory_order_acquire); + uint32_t tail = atomic_load_explicit(&control->tail, memory_order_relaxed); + if ((uint32_t) (tail - head) >= SHIM_MMAP_RING_SIZE) { + el1_gate_leave(context); + el1_mmap_counter(context, SHIM_MMAP_COUNTER_RING_FULL); + return false; + } + + uint64_t address = + atomic_load_explicit(&control->cursor, memory_order_relaxed); + uint64_t cursor; + if (length >= EL1_ALIGN_THRESHOLD) { + if (el1_add_overflow(address, EL1_BLOCK_SIZE - 1, &address)) { + el1_gate_leave(context); + el1_mmap_counter(context, SHIM_MMAP_COUNTER_CAPACITY_MISS); + return false; + } + address &= ~(EL1_BLOCK_SIZE - 1); + } + if (el1_add_overflow(address, length, &cursor) || + cursor > + atomic_load_explicit(&control->arena_limit, memory_order_relaxed)) { + el1_gate_leave(context); + el1_mmap_counter(context, SHIM_MMAP_COUNTER_CAPACITY_MISS); + return false; + } + + shim_mmap_entry_t *entry = &control->ring[tail & (SHIM_MMAP_RING_SIZE - 1)]; + entry->addr = address; + entry->len = length; + entry->prot = prot; + /* publish the bump cursor before the entry */ + atomic_store_explicit(&control->cursor, cursor, memory_order_relaxed); + atomic_store_explicit(&control->tail, tail + 1, memory_order_release); + atomic_store_explicit(&control->pending_work, 1, memory_order_release); + + el1_gate_leave(context); + saved_gprs[0] = address; + el1_mmap_counter(context, SHIM_MMAP_COUNTER_HIT); + return true; +} + +/* munmap: invalidate an anonymous arena range synchronously at EL1, then + * release-publish a metadata retirement to this vCPU's SPSC ring. The host + * consumes it at a later natural VM exit. No shared allocator cursor is + * modified here. + * + * Validation precedes mutation, so clearing cannot discover a need to split + * after changing an earlier descriptor. A range within one L3 table retains the + * validated table pointer; other shapes use a second walk. Whole L2 blocks and + * L3 leaves are the only descriptors this path writes. + */ + +/* generation is re-read after bounds so a descriptor publication cannot be + * observed torn. + */ +static bool el1_munmap_match_arena(shim_mmap_control_t *candidate, + uint64_t start, + uint64_t end, + unsigned slot, + uint32_t *generation_out, + uint32_t *flags_out, + unsigned *slot_out) +{ + uint32_t flags = + atomic_load_explicit(&candidate->flags, memory_order_acquire); + if (!(flags & SHIM_MMAP_CTRL_ENABLED)) + return false; + uint32_t generation = + atomic_load_explicit(&candidate->generation, memory_order_acquire); + if (start < + atomic_load_explicit(&candidate->arena_base, memory_order_relaxed)) + return false; + if (end > atomic_load_explicit(&candidate->cursor, memory_order_acquire)) + return false; + if (generation != + atomic_load_explicit(&candidate->generation, memory_order_acquire)) + return false; + *generation_out = generation; + *flags_out = flags; + *slot_out = slot; + return true; +} + +/* Locate the one arena generation containing the whole range. Owner-local + * munmap normally stops at context->control; cross-vCPU retirement scans the + * remaining descriptors. + */ +static shim_mmap_control_t *el1_munmap_find_arena( + const el1_mmap_context_t *context, + uint64_t start, + uint64_t end, + uint32_t *generation_out, + uint32_t *flags_out, + unsigned *slot_out) +{ + if (el1_munmap_match_arena(context->control, start, end, context->slot, + generation_out, flags_out, slot_out)) + return context->control; + + for (unsigned slot = 0; slot < EL1_ARENA_SLOTS; slot++) { + if (slot == context->slot) + continue; + shim_mmap_control_t *candidate = + (shim_mmap_control_t *) (context->shim_data + + SHIM_MMAP_CONTROL_BASE + + (uintptr_t) slot * + SHIM_MMAP_CONTROL_STRIDE); + if (el1_munmap_match_arena(candidate, start, end, slot, generation_out, + flags_out, slot_out)) + return candidate; + } + return NULL; +} + +static inline uint64_t el1_vale_operand(uint64_t address, unsigned level) +{ + uint64_t base = (address >> 12) & ((1ULL << 44) - 1); + if (level == 0) + return base; + /* TTL[3:2]=01 selects the 4 KiB granule; TTL[1:0] selects the level. */ + return base | ((uint64_t) (4u | level) << 44); +} + +static inline uint64_t el1_rvale_operand(uint64_t address, + uint64_t units, + unsigned scale, + unsigned level) +{ + uint64_t base = (address >> 12) & ((1ULL << 37) - 1); + return base | ((uint64_t) level << 37) | ((units - 1) << 39) | + ((uint64_t) scale << 44) | (1ULL << 46); +} + +/* Encode SCALE=0..3 RVALE1IS batches directly from the changed envelope. Each + * instruction covers up to 32 scale units: 64 pages at SCALE=0, 2048 at + * SCALE=1, 65536 at SCALE=2, and 2097152 (8 GiB) at SCALE=3. Widening to the + * scale-unit boundary only invalidates neighboring TLB entries; it cannot + * change mappings. Ranges larger than one instruction are emitted as adjacent + * batches while retaining a single completion DSB. + */ +static void el1_tlbi_pages(uint64_t start, + uint64_t end, + bool range_supported, + unsigned level) +{ + uint64_t pages = (end - start) / EL1_PAGE_SIZE; + if (pages <= 1 || (!range_supported && pages <= 8)) { + /* The Armv8.4+ range flag also establishes FEAT_TTL. */ + unsigned ttl_level = range_supported ? level : 0; + uint64_t operand = el1_vale_operand(start, ttl_level); + for (uint64_t page = 0; page < pages; page++) + el1_tlbi_vale1is(operand + page); + return; + } + + if (!range_supported) { + el1_tlbi_all(); + return; + } + + unsigned scale; + unsigned unit_shift; + if (pages <= 64) { + scale = 0; + unit_shift = 13; /* 2 pages = 8 KiB */ + } else if (pages <= 0x800) { + scale = 1; + unit_shift = 18; /* 64 pages = 256 KiB */ + } else if (pages <= 0x10000) { + scale = 2; + unit_shift = 23; /* 2048 pages = 8 MiB */ + } else { + scale = 3; + unit_shift = 28; /* 65536 pages = 256 MiB */ + } + + uint64_t unit = 1ULL << unit_shift; + uint64_t mask = unit - 1; + uint64_t first = start & ~mask; + uint64_t last; + if (el1_add_overflow(end, mask, &last)) { + el1_tlbi_all(); + return; + } + last &= ~mask; + while (first < last) { + uint64_t units = (last - first) >> unit_shift; + if (units > 32) + units = 32; + el1_tlbi_rvale1is(el1_rvale_operand(first, units, scale, level)); + first += units << unit_shift; + } +} + +typedef enum { + EL1_VALIDATE_GENERIC, /* walk per block, mixed L2/L3 shapes */ + EL1_VALIDATE_L2_BLOCKS, /* proven L2-block-only under one L0 slot */ + EL1_VALIDATE_BAD, /* corrupt or unsupported descriptor shape */ +} el1_validate_t; + +typedef struct { + _Atomic uint64_t *single_l3; + unsigned single_l3_first; + unsigned single_l3_limit; + unsigned tlbi_level; +} el1_munmap_pt_plan_t; + +/* Common large-anonymous case: the materialized envelope is 2 MiB aligned and + * consists only of L2 block descriptors (or holes retired by a sibling). Walk + * L0 once, then L1 once per 1 GiB window and scan the L2 entries linearly. The + * generic path remains the fallback for any L3 table or unusual upper shape. + */ +static el1_validate_t el1_validate_l2_blocks(uint64_t start, uint64_t end) +{ + if (((start | end) & (EL1_BLOCK_SIZE - 1)) != 0) + return EL1_VALIDATE_GENERIC; + if ((((end - 1) ^ start) >> 39) != 0) /* must stay in one L0 slot */ + return EL1_VALIDATE_GENERIC; + + _Atomic uint64_t *l0 = el1_table(el1_read_ttbr0()); + uint64_t l0_desc = + atomic_load_explicit(&l0[(start >> 39) & 0x1ff], memory_order_acquire); + uint64_t type = el1_desc_type(l0_desc); + if (type == 0) + return EL1_VALIDATE_L2_BLOCKS; + if (type != EL1_DESC_TABLE) + return EL1_VALIDATE_GENERIC; + + _Atomic uint64_t *l1 = el1_table(l0_desc); + uint64_t address = start; + while (address < end) { + uint64_t window_end = + el1_min(((address >> 30) + 1) << 30, end); /* next 1 GiB */ + uint64_t l1_desc = atomic_load_explicit(&l1[(address >> 30) & 0x1ff], + memory_order_acquire); + uint64_t l1_type = el1_desc_type(l1_desc); + if (l1_type == 0) { + address = window_end; + continue; + } + if (l1_type != EL1_DESC_TABLE) + return EL1_VALIDATE_GENERIC; + + _Atomic uint64_t *l2 = el1_table(l1_desc); + unsigned index = (unsigned) ((address >> 21) & 0x1ff); + while (address < window_end) { + uint64_t l2_desc = + atomic_load_explicit(&l2[index], memory_order_relaxed); + uint64_t l2_type = el1_desc_type(l2_desc); + if (l2_type != 0) { + if (l2_type != EL1_DESC_BLOCK) /* L2 block, never L3 table */ + return EL1_VALIDATE_GENERIC; + } + index++; + address += EL1_BLOCK_SIZE; + } + } + return EL1_VALIDATE_L2_BLOCKS; +} + +/* Validation records a single-L3 clear range and the known leaf level. The + * caller charges the materialized-envelope intersection as a conservative upper + * bound rather than reading every L3 leaf merely to count exact bytes. + */ +static el1_validate_t el1_validate_generic(uint64_t request_start, + uint64_t request_end, + uint64_t start, + uint64_t end, + el1_munmap_pt_plan_t *plan) +{ + bool saw_l2 = false; + bool saw_l3 = false; + bool single_block = el1_block_start(start) == el1_block_start(end - 1); + uint64_t address = start; + while (address < end) { + _Atomic uint64_t *l0 = el1_table(el1_read_ttbr0()); + uint64_t l0_desc = atomic_load_explicit(&l0[(address >> 39) & 0x1ff], + memory_order_acquire); + if (el1_desc_type(l0_desc) == 0) { + address = el1_next_block(address); + continue; + } + if (el1_desc_type(l0_desc) != EL1_DESC_TABLE) + return EL1_VALIDATE_BAD; + + _Atomic uint64_t *l1 = el1_table(l0_desc); + uint64_t l1_desc = atomic_load_explicit(&l1[(address >> 30) & 0x1ff], + memory_order_acquire); + if (el1_desc_type(l1_desc) == 0) { + address = el1_next_block(address); + continue; + } + if (el1_desc_type(l1_desc) != EL1_DESC_TABLE) + return EL1_VALIDATE_BAD; + + _Atomic uint64_t *l2 = el1_table(l1_desc); + uint64_t l2_desc = atomic_load_explicit(&l2[(address >> 21) & 0x1ff], + memory_order_acquire); + uint64_t l2_type = el1_desc_type(l2_desc); + if (l2_type == 0) { + address = el1_next_block(address); + continue; + } + if (l2_type == EL1_DESC_TABLE) { + saw_l3 = true; + uint64_t block_end = el1_min(el1_next_block(address), end); + if (single_block) { + plan->single_l3 = el1_table(l2_desc); + plan->single_l3_first = (unsigned) ((address >> 12) & 0x1ff); + plan->single_l3_limit = + plan->single_l3_first + + (unsigned) ((block_end - address) / EL1_PAGE_SIZE); + } + address = block_end; + continue; + } + if (l2_type != EL1_DESC_BLOCK) + return EL1_VALIDATE_BAD; + + saw_l2 = true; + /* An L2 block can only be cleared as a whole. */ + uint64_t block_start = el1_block_start(address); + if (request_start > block_start || + request_end < block_start + EL1_BLOCK_SIZE) + return EL1_VALIDATE_BAD; + address = block_start + EL1_BLOCK_SIZE; + } + if (saw_l2 != saw_l3) + plan->tlbi_level = saw_l2 ? EL1_TLBI_LEVEL_L2 : EL1_TLBI_LEVEL_L3; + return EL1_VALIDATE_GENERIC; +} + +/* Mutation pass. Host writers are gated; sibling fast munmaps can only change + * the same descriptors to zero, so repeated invalidation is safe. + */ +static inline void el1_clear_l3_batch(_Atomic uint64_t *entry) +{ + __asm__ volatile( + "stp xzr, xzr, [%0, #0]\n\t" + "stp xzr, xzr, [%0, #16]\n\t" + "stp xzr, xzr, [%0, #32]\n\t" + "stp xzr, xzr, [%0, #48]" + : + : "r"(entry) + : "memory"); +} + +static inline void el1_clear_l3_pair(_Atomic uint64_t *entry) +{ + __asm__ volatile("stp xzr, xzr, [%0]" : : "r"(entry) : "memory"); +} + +static void el1_clear_l3_entries(_Atomic uint64_t *l3, + unsigned first, + unsigned limit) +{ + /* Page tables are page-aligned. Peel to a 64-byte group with scalar and + * paired stores, clear full groups, then clear paired and scalar tails. The + * caller's DSB ISHST orders every descriptor write before TLBI. + */ + while (first < limit && (first & (EL1_L3_PTES_PER_BATCH - 1)) != 0) { + if (((first & 1u) == 0) && (limit - first >= 2)) { + el1_clear_l3_pair(&l3[first]); + first += 2; + } else { + atomic_store_explicit(&l3[first], 0, memory_order_relaxed); + first++; + } + } + while (limit - first >= EL1_L3_PTES_PER_BATCH) { + el1_clear_l3_batch(&l3[first]); + first += EL1_L3_PTES_PER_BATCH; + } + while (limit - first >= 2) { + el1_clear_l3_pair(&l3[first]); + first += 2; + } + if (first < limit) + atomic_store_explicit(&l3[first], 0, memory_order_relaxed); +} + +static bool el1_clear_generic(uint64_t start, uint64_t end) +{ + uint64_t address = start; + while (address < end) { + _Atomic uint64_t *l0 = el1_table(el1_read_ttbr0()); + uint64_t l0_desc = atomic_load_explicit(&l0[(address >> 39) & 0x1ff], + memory_order_acquire); + if (el1_desc_type(l0_desc) != EL1_DESC_TABLE) { + address = el1_next_block(address); + continue; + } + + _Atomic uint64_t *l1 = el1_table(l0_desc); + uint64_t l1_desc = atomic_load_explicit(&l1[(address >> 30) & 0x1ff], + memory_order_acquire); + if (el1_desc_type(l1_desc) != EL1_DESC_TABLE) { + address = el1_next_block(address); + continue; + } + + _Atomic uint64_t *l2 = el1_table(l1_desc); + _Atomic uint64_t *l2_entry = &l2[(address >> 21) & 0x1ff]; + uint64_t l2_desc = atomic_load_explicit(l2_entry, memory_order_acquire); + uint64_t l2_type = el1_desc_type(l2_desc); + if (l2_type == 0) { + address = el1_next_block(address); + continue; + } + if (l2_type == EL1_DESC_BLOCK) { + atomic_store_explicit(l2_entry, 0, memory_order_release); + address = el1_next_block(address); + continue; + } + if (l2_type != EL1_DESC_TABLE) + return false; /* validation invariant */ + + _Atomic uint64_t *l3 = el1_table(l2_desc); + uint64_t block_end = el1_min(el1_next_block(address), end); + unsigned first = (unsigned) ((address >> 12) & 0x1ff); + unsigned limit = + first + (unsigned) ((block_end - address) / EL1_PAGE_SIZE); + el1_clear_l3_entries(l3, first, limit); + address = block_end; + } + return true; +} + +/* Validation proved one L0 table and L2-block-only leaves. Re-walk L1 once per + * 1 GiB window, then use ordinary aligned 64-bit stores. Those stores are + * atomic; the DSB ISHST below supplies the required publication order, so a + * release store on every descriptor is unnecessary. + */ +static void el1_clear_l2_blocks(uint64_t start, uint64_t end) +{ + _Atomic uint64_t *l0 = el1_table(el1_read_ttbr0()); + uint64_t l0_desc = + atomic_load_explicit(&l0[(start >> 39) & 0x1ff], memory_order_relaxed); + if (el1_desc_type(l0_desc) != EL1_DESC_TABLE) + return; + _Atomic uint64_t *l1 = el1_table(l0_desc); + + uint64_t address = start; + while (address < end) { + uint64_t window_end = el1_min(((address >> 30) + 1) << 30, end); + uint64_t l1_desc = atomic_load_explicit(&l1[(address >> 30) & 0x1ff], + memory_order_relaxed); + if (el1_desc_type(l1_desc) == 0) { + address = window_end; + continue; + } + + _Atomic uint64_t *l2 = el1_table(l1_desc); + unsigned index = (unsigned) ((address >> 21) & 0x1ff); + while (address < window_end) { + /* Validation plus the closed host gate proves this slot is a block + * or already zero. A sibling producer can only move it toward zero, + * so an unconditional aligned store is safe and avoids a second + * load/branch. + */ + atomic_store_explicit(&l2[index], 0, memory_order_relaxed); + index++; + address += EL1_BLOCK_SIZE; + } + } +} + +static bool el1_munmap(el1_mmap_context_t *context, uint64_t *saved_gprs) +{ + shim_mmap_control_t *control = context->control; + munmap_retire_ring_t *retire = &control->retire; + + if (el1_attention_pending(context)) { + el1_attention_counter(context); + return false; + } + + uint64_t start = saved_gprs[0]; + uint64_t length = saved_gprs[1]; + uint64_t end; + if (length == 0 || (start & (EL1_PAGE_SIZE - 1)) || + (length & (EL1_PAGE_SIZE - 1)) || el1_add_overflow(start, length, &end)) + return false; + + uint32_t head = atomic_load_explicit(&retire->head, memory_order_acquire); + uint32_t tail = atomic_load_explicit(&retire->tail, memory_order_relaxed); + if ((uint32_t) (tail - head) >= SHIM_MUNMAP_RETIRE_RING_SIZE - 1) { + atomic_fetch_add_explicit(&control->munmap_retire_near_full, 1, + memory_order_relaxed); + return false; /* near-full => batched HVC drain */ + } + uint64_t produced = + atomic_load_explicit(&retire->produced_bytes, memory_order_relaxed); + + switch (el1_gate_enter(context)) { + case EL1_GATE_ENTERED: + break; + case EL1_GATE_CLOSED_BEFORE_ANNOUNCE: + case EL1_GATE_CLOSED_AFTER_ANNOUNCE: + return false; + } + + uint32_t generation; + uint32_t arena_flags; + unsigned arena_slot; + shim_mmap_control_t *owner = el1_munmap_find_arena( + context, start, end, &generation, &arena_flags, &arena_slot); + if (!owner) { + el1_gate_leave(context); + return false; + } + + /* Reserve and fill, but do not advance tail until PTE+TLBI completion. */ + munmap_retire_entry_t *entry = + &retire->entries[tail & (SHIM_MUNMAP_RETIRE_RING_SIZE - 1)]; + entry->addr = start; + entry->length = length; + entry->arena_generation = generation; + entry->flags = arena_slot; + + /* Refill invalidates every stale descriptor before publishing a new + * generation. Until the host records a lazy materialization in that + * generation, the entire arena is known PTE-empty and no walk/TLBI is + * needed. The host PT gate makes the generation marker stable here. + */ + uint64_t charged_pages = 0; + uint64_t walk_start = 0; + uint64_t walk_end = 0; + bool walked = false; + if (atomic_load_explicit(&owner->materialized_generation, + memory_order_acquire) == generation) { + uint64_t materialized_start = atomic_load_explicit( + &owner->materialized_start, memory_order_relaxed); + uint64_t materialized_end = atomic_load_explicit( + &owner->materialized_end, memory_order_relaxed); + walk_start = start > materialized_start ? start : materialized_start; + walk_end = el1_min(end, materialized_end); + walked = walk_start < walk_end; + } + + bool l2_blocks = false; + el1_munmap_pt_plan_t plan = {0}; + if (walked) { + el1_validate_t verdict = el1_validate_l2_blocks(walk_start, walk_end); + if (verdict == EL1_VALIDATE_L2_BLOCKS) { + l2_blocks = true; + plan.tlbi_level = EL1_TLBI_LEVEL_L2; + } else { + verdict = + el1_validate_generic(start, end, walk_start, walk_end, &plan); + if (verdict == EL1_VALIDATE_BAD) { + el1_gate_leave(context); + return false; + } + } + + /* Charge the materialized-envelope intersection rather than walking + * every L3 leaf to count exact PTE coverage. Holes can only make this + * an overestimate, which is safe for the advisory cleanup threshold; + * the host consumes the same bound from this retirement entry. + */ + charged_pages = (walk_end - walk_start) / EL1_PAGE_SIZE; + entry->flags = + (uint32_t) (charged_pages << SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT) | + (uint32_t) arena_slot; + } + + if (walked && charged_pages != 0) { + uint64_t consumed = + atomic_load_explicit(&retire->consumed_bytes, memory_order_acquire); + uint64_t charge = charged_pages << 12; + uint64_t pending; + if (el1_add_overflow(produced - consumed, charge, &pending) || + el1_add_overflow(produced, charge, &produced)) { + el1_gate_leave(context); + return false; + } + + /* Crossing the soft threshold records cleanup_requested but never + * forces this producer to HVC; another natural exit consumes the ring. + */ + if (pending > SHIM_MUNMAP_RETIRE_BYTES_SOFT) + atomic_store_explicit(&retire->cleanup_requested, 1, + memory_order_release); + + if (l2_blocks) { + el1_clear_l2_blocks(walk_start, walk_end); + } else if (plan.single_l3) { + el1_clear_l3_entries(plan.single_l3, plan.single_l3_first, + plan.single_l3_limit); + } else if (!el1_clear_generic(walk_start, walk_end)) { + el1_gate_leave(context); + return false; + } + + /* Descriptor stores -> leaf invalidation -> completed visibility before + * the retire tail release makes metadata cleanup eligible. + */ + el1_dsb_ishst(); + el1_tlbi_pages(walk_start, walk_end, + (arena_flags & SHIM_MMAP_CTRL_TLBIRANGE) != 0, + plan.tlbi_level); + el1_dsb_ish(); + el1_isb(); + } + + atomic_store_explicit(&retire->produced_bytes, produced, + memory_order_relaxed); + atomic_store_explicit(&retire->tail, tail + 1, memory_order_release); + atomic_store_explicit(&context->control->pending_work, 1, + memory_order_release); + el1_gate_leave(context); + saved_gprs[0] = 0; + return true; +} + +bool el1_mmap_fastpath(uint64_t saved_gprs[static EL1_SAVED_GPRS]) +{ + el1_mmap_context_t context = el1_context(saved_gprs); + switch (saved_gprs[8]) { + case EL1_SYS_MMAP: + return el1_mmap(&context, saved_gprs); + case EL1_SYS_MUNMAP: + return el1_munmap(&context, saved_gprs); + default: + return false; + } +} diff --git a/src/core/shim-mmap.h b/src/core/shim-mmap.h new file mode 100644 index 00000000..36b5850e --- /dev/null +++ b/src/core/shim-mmap.h @@ -0,0 +1,20 @@ +/* + * Freestanding EL1 mmap-family fast paths. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The assembly exception shim owns the saved-register frame and calls this + * module only for mmap-family syscalls. A true return means X0 in the saved + * frame contains the completed syscall result; false asks the shim to forward + * the original frame to HVC #5 unchanged. + */ + +#pragma once + +#include +#include + +#define EL1_SAVED_GPRS 31u + +bool el1_mmap_fastpath(uint64_t saved_gprs[static EL1_SAVED_GPRS]); diff --git a/src/core/shim.S b/src/core/shim.S index 97425197..bc3c566f 100644 --- a/src/core/shim.S +++ b/src/core/shim.S @@ -27,7 +27,7 @@ * 4 = single-shot TLBI RVAE1IS (FEAT_TLBIRANGE); * X9 carries the pre-encoded RVAE1IS operand * (baddr | NUM<<39 | SCALE<<44 | TTL<<37 | - * ASID<<48; SCALE=0, TTL=0, ASID=0 today) + * ASID<<48; TTL=0 and ASID=0) * X11 carries the I-cache hint for X8 in {1, 3, 4}: * 1 = IC IALLU after the TLBI sequence (new * executable content visible to EL0), 0 = skip the @@ -449,8 +449,24 @@ svc_handler: b.eq getsid_fast cmp x10, #278 /* SYS_getrandom? */ b.eq getrandom_fast + cmp x10, #215 /* SYS_munmap? */ + b.eq mmap_family_fast + cmp x10, #222 /* SYS_mmap? */ + b.eq mmap_family_fast b handle_svc_0 +/* mmap and munmap share a freestanding C consumer. The saved frame remains + * authoritative: false leaves it untouched for HVC #5; true places the + * completed result in saved X0. The normal restore tail reloads every other + * guest register, so the C ABI's caller-clobbered set is private to EL1. + */ +mmap_family_fast: + mov x0, sp + bl _el1_mmap_fastpath + cbz w0, handle_svc_0 + ldr x0, [sp, #0] + b svc_restore_eret + identity_class_fast: mrs x12, tpidr_el1 /* shim-globals base */ ldar w13, [x12] /* attention flag, acquire */ @@ -1206,6 +1222,7 @@ handle_el0_fault: .Lel0_fault_tlbi_full: /* Broadcast TLB + conditional I-cache flush. X11=0 skips IC IALLU. */ + dsb ishst tlbi vmalle1is dsb ish cbz x11, .Lel0_fault_full_no_ic @@ -1227,6 +1244,7 @@ handle_el0_fault: mov x13, x11 ubfx x11, x9, #12, #44 mov x12, x10 + dsb ishst 4: tlbi vae1is, x11 add x11, x11, #1 subs x12, x12, #1 @@ -1241,7 +1259,10 @@ handle_el0_fault: .Lel0_fault_tlbi_rvae: /* Single-shot TLBI RVAE1IS (FEAT_TLBIRANGE). X9 carries the pre-encoded - * operand (baddr | NUM<<39 | TG=01<<46); X11 the I-cache hint. */ + * operand (baddr | NUM<<39 | SCALE<<44 | TG=01<<46); X11 the I-cache + * hint. + */ + dsb ishst tlbi rvae1is, x9 dsb ish cbz x11, .Lel0_fault_rvae_no_ic @@ -1285,6 +1306,7 @@ tlbi_restore_eret: * into the TTL [47:44] or ASID [63:48] operand fields. */ ubfx x0, x0, #12, #44 + dsb ishst tlbi vae1is, x0 dsb ish ic iallu @@ -1358,6 +1380,7 @@ handle_svc_0: * 1 = broadcast TLBI VMALLE1IS * 2 = execve replaced register state (drop frame + flush) * 3 = selective TLBI VAE1IS over X10 pages starting at X9 + * 4 = single-shot TLBI RVAE1IS with encoded operand in X9 * 5. Resume vCPU (execution continues below) */ hvc #5 @@ -1386,6 +1409,7 @@ tlbi_full: * the shim must IC IALLU; zero means a data-only PT change and the I-cache * invalidation is skipped. */ + dsb ishst tlbi vmalle1is dsb ish cbz x11, .Ltlbi_full_skip_ic @@ -1422,6 +1446,7 @@ tlbi_selective: */ ubfx x11, x9, #12, #44 /* x11 = VA[55:12] (current page operand) */ mov x12, x10 /* x12 = remaining page counter */ + dsb ishst 3: tlbi vae1is, x11 add x11, x11, #1 /* next page (operand is in 4 KiB units) */ subs x12, x12, #1 @@ -1437,11 +1462,11 @@ tlbi_selective: tlbi_range_large: /* Single-shot TLBI RVAE1IS (FEAT_TLBIRANGE, ARMv8.4+). The host has encoded * the full operand in X9: baddr (VA >> 12), TTL=0, NUM in bits [43:39], - * SCALE=0, ASID=0. One instruction covers up to 64 pages, avoiding the - * broadcast TLBI VMALLE1IS that the prior selective cap forced for - * 17..64-page ranges. X11 carries the I-cache hint as in tlbi_full / + * SCALE in bits [45:44], ASID=0. One instruction covers up to + * TLBI_RVAE_MAX_PAGES. X11 carries the I-cache hint as in tlbi_full / * tlbi_selective. */ + dsb ishst tlbi rvae1is, x9 dsb ish cbz x11, .Ltlbi_rvae_skip_ic diff --git a/src/proved/align.h b/src/proved/align.h index 42608aea..b1eda0d6 100644 --- a/src/proved/align.h +++ b/src/proved/align.h @@ -60,7 +60,7 @@ ensures binary: \result == 0 || \result == 1; ensures rejects_only_on_wrap: \result != 0 <==> (x % align == 0 - || (x / align + 1) * align <= UINT64_MAX); + || x / align < UINT64_MAX / align); ensures aligned: \result != 0 ==> (\exists integer k; *out == k * align); ensures never_below: \result != 0 ==> *out >= x; diff --git a/src/proved/mmap-fastpath.h b/src/proved/mmap-fastpath.h new file mode 100644 index 00000000..93b8f935 --- /dev/null +++ b/src/proved/mmap-fastpath.h @@ -0,0 +1,236 @@ +/* + * EL1 mmap fast-path arena sizing and capacity arithmetic: the parts a proof + * can reach + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Each per-vCPU arena is a bump allocator: EL1 hands out [cursor, cursor+len) + * and advances cursor until a request no longer fits, at which point the host + * refills or grows the arena from the guest's own mmap history. Every value + * here is guest-influenced (request_len is the guest's mmap length, window_max + * is derived from a history of guest-chosen lengths), so a slip either wedges + * the allocator (undersizes an arena forever) or, in + * mmap_fastpath_request_fits, accepts a request that runs past arena_limit. + * + * Split out of mem.c because mem.c cannot be given to Frama-C: it includes + * sys/mman.h and the HVF headers, which the analyzer's libc does not model. + * These functions need nothing but stdint.h plus proved/align.h, so make + * verify-mmapfastpath proves this header directly. + * + * MMAP_FAST_ARENA_MIN/MAX/TARGET_ENTRIES and MMAP_FAST_PUBLICATION_WINDOW live + * here rather than in core/mmap-fastpath.h so the sizing policy has one + * definition; core/mmap-fastpath.h includes this header for them rather than + * repeating the numbers where a proof cannot see whether they still match. + */ + +#pragma once + +#include +#include + +#include "proved/align.h" + +#define MMAP_FAST_ARENA_MIN (64ULL * 1024 * 1024) +#define MMAP_FAST_ARENA_MAX (32ULL * 1024 * 1024 * 1024) +#define MMAP_FAST_ARENA_TARGET_ENTRIES 32u + +/* Registrations the arena sizer looks back over. Sixteen is long enough to + * outlive a burst of one allocation size (so a phase change does not resize the + * arena on its first mapping) and short enough that an outlier washes out + * within a few dozen mappings. + */ +#define MMAP_FAST_PUBLICATION_WINDOW 16u + +/* Whether a request of len bytes still fits the bump cursor before limit. + * + * Requests at least block_align wide are served block-aligned, matching the + * fast path's block-granular VA carve (guest_invalidate_ptes clears a whole + * block, so a sub-block bump allocation inside a block already carrying a live + * mapping would straddle stale and fresh PTEs). Requests below that threshold + * draw straight from cursor. + * + * len == 0 is its own case rather than falling out of the general fits test: + * mmap_fastpath_topup_locked calls this with len == 0 to ask "is there any room + * left", and an arena with cursor == limit has none, so that case needs cursor + * < limit rather than window_fits's cursor <= limit (which would call a + * completely full arena still fitting a zero-length request). + * + * Only the len >= block_align case is stated one-directionally (\result ==> + * room existed at the unaligned cursor, not the converse): its true value comes + * from align_up_ok, which bounds the aligned start (never_below, + * rounds_up_once) rather than pinning it to a closed-form expression restated + * here. That is enough to rule out the dangerous direction -- this function + * reporting a fit that ends up past limit -- and chasing the converse would + * mean re-deriving align_up_ok's own arithmetic in this contract instead of + * resting on its proof. + */ +/*@ + requires block_align > 0; + assigns \nothing; + ensures zero_len: + len == 0 ==> (\result <==> cursor < limit); + ensures small_request: + (len != 0 && len < block_align) ==> + (\result <==> (cursor <= limit && len <= limit - cursor)); + ensures large_request_sound: + (len != 0 && len >= block_align && \result) ==> + (cursor <= limit && len <= limit - cursor); + */ +static inline bool mmap_fastpath_request_fits(uint64_t cursor, + uint64_t limit, + uint64_t len, + uint64_t block_align) +{ + if (!len) + return cursor < limit; + uint64_t start = cursor; + if (len >= block_align && !align_up_ok(cursor, block_align, &start)) + return false; + return window_fits(start, len, limit); +} + +/* Ratio the doubling loop below relies on: MMAP_FAST_ARENA_MAX is exactly + * MMAP_FAST_ARENA_MIN doubled 9 times. Guards the loop's trip-count bound + * against either constant changing without updating the other. + */ +_Static_assert( + MMAP_FAST_ARENA_MAX == MMAP_FAST_ARENA_MIN * 512, + "mmap_fastpath_pow2_clamped's loop bound assumes MAX == MIN << 9"); + +/*@ + axiomatic ArenaPow2 { + logic integer arena_pow2(integer k); + axiom arena_pow2_zero: arena_pow2(0) == 1; + axiom arena_pow2_succ: + \forall integer k; k >= 0 ==> arena_pow2(k + 1) == 2 * arena_pow2(k); + } +*/ + +/* Smallest power of two in [MMAP_FAST_ARENA_MIN, MMAP_FAST_ARENA_MAX] that is + * at least value, or the nearer bound when value falls outside that range. + * + * This used to be the classic bit-smear round-up-to-power-of-two (value--; + * value |= value >> 1; ...; return value + 1;). Every claim reaching through + * that form -- including a bound as simple as \result >= MMAP_FAST_ARENA_MIN -- + * turned out to be a bitvector-shaped goal (OR only sets bits, so value | value + * >> k >= value for any k, but that is a property of the bit pattern, not of + * linear arithmetic), and neither alt-ergo nor z3 discharges even a single OR + * step of it here; proved/align.h's own history is the same shape one level + * down. Same semantics, different algorithm: this doubles from + * MMAP_FAST_ARENA_MIN instead, which is linear arithmetic all the way through + * and discharges completely, including the power-of-two and covering properties + * the smear form could not get past its two boundary branches. It costs at most + * 9 iterations (see the _Static_assert above) where the smear was 6 fixed + * shifts; both run once per arena refill, not per mmap, so the difference does + * not reach the hot path this feeds. + */ +/*@ + assigns \nothing; + ensures bounded_low: \result >= MMAP_FAST_ARENA_MIN; + ensures bounded_high: \result <= MMAP_FAST_ARENA_MAX; + ensures covers: + value <= MMAP_FAST_ARENA_MAX ==> \result >= value; + ensures clamp_low: + value <= MMAP_FAST_ARENA_MIN ==> \result == MMAP_FAST_ARENA_MIN; + ensures clamp_high: + value >= MMAP_FAST_ARENA_MAX ==> \result == MMAP_FAST_ARENA_MAX; + */ +static inline uint64_t mmap_fastpath_pow2_clamped(uint64_t value) +{ + if (value <= MMAP_FAST_ARENA_MIN) + return MMAP_FAST_ARENA_MIN; + if (value >= MMAP_FAST_ARENA_MAX) + return MMAP_FAST_ARENA_MAX; + uint64_t p = MMAP_FAST_ARENA_MIN; + /*@ ghost int k = 0; */ + /*@ + loop invariant p_eq: p == MMAP_FAST_ARENA_MIN * arena_pow2(k); + loop invariant k_range: 0 <= k <= 9; + loop assigns p, k; + loop variant 9 - k; + */ + while (p < value) { + p += p; + /*@ ghost k = k + 1; */ + } + return p; +} + +/* Largest registration still inside a fixed-size history window. window must + * point at MMAP_FAST_PUBLICATION_WINDOW live entries; the caller passes + * shim_mmap_control_t's publication_window array, which this header does not + * name so it never has to see that struct's _Atomic fields. + * + * Only the upper bound is stated, not that \result is itself one of the + * entries: a ghost witness index tracking argmax alongside max proves its own + * base case and the two ensures it would support (both go through by taking the + * invariant as given), but the loop-invariant preservation step itself times + * out under both alt-ergo and z3 -- at 90s, ten times FRAMAC_TIMEOUT, not just + * 30s -- while the plain upper-bound invariant below proves in milliseconds. + * The upper bound is also the only property arena_size's sizing math actually + * needs. + */ +/*@ + requires \valid_read(window + (0 .. MMAP_FAST_PUBLICATION_WINDOW - 1)); + assigns \nothing; + ensures upper_bound: + \forall integer i; 0 <= i < MMAP_FAST_PUBLICATION_WINDOW ==> + window[i] <= \result; + */ +static inline uint64_t mmap_fastpath_window_max( + const uint64_t window[MMAP_FAST_PUBLICATION_WINDOW]) +{ + uint64_t max = 0; + /*@ + loop invariant bound: 0 <= i <= MMAP_FAST_PUBLICATION_WINDOW; + loop invariant prefix_le_max: + \forall integer j; 0 <= j < i ==> window[j] <= max; + loop assigns i, max; + loop variant MMAP_FAST_PUBLICATION_WINDOW - i; + */ + for (unsigned i = 0; i < MMAP_FAST_PUBLICATION_WINDOW; i++) + if (window[i] > max) + max = window[i]; + return max; +} + +/* Target arena size covering both the recent registration history (window_max) + * and the request that is about to be served (request_len), clamped to + * [MMAP_FAST_ARENA_MIN, MMAP_FAST_ARENA_MAX]. + * + * The bound ensures below rest on mmap_fastpath_pow2_clamped's own + * bounded_low/bounded_high: adaptive and covering are each either + * MMAP_FAST_ARENA_MIN (the guard's false branch) or a pow2_clamped result, both + * now fully covered by that function's contract. -wp-rte separately proves this + * function's own arithmetic sound regardless: the two multiplication guards + * (window_max > MAX / target_entries, request_len > MAX / 2) mean neither + * window_max * target_entries nor request_len * 2 can overflow, and there is no + * division-by-zero. + */ +/*@ + assigns \nothing; + ensures result_ge_min: \result >= MMAP_FAST_ARENA_MIN; + ensures result_le_max: \result <= MMAP_FAST_ARENA_MAX; + */ +static inline uint64_t mmap_fastpath_arena_size(uint64_t window_max, + uint64_t request_len) +{ + uint64_t adaptive = MMAP_FAST_ARENA_MIN; + if (window_max) { + const uint64_t target_entries = MMAP_FAST_ARENA_TARGET_ENTRIES; + uint64_t target = window_max > MMAP_FAST_ARENA_MAX / target_entries + ? MMAP_FAST_ARENA_MAX + : window_max * target_entries; + adaptive = mmap_fastpath_pow2_clamped(target); + } + + uint64_t covering = MMAP_FAST_ARENA_MIN; + if (request_len) { + uint64_t target = request_len > MMAP_FAST_ARENA_MAX / 2 + ? MMAP_FAST_ARENA_MAX + : request_len * 2; + covering = mmap_fastpath_pow2_clamped(target); + } + return adaptive > covering ? adaptive : covering; +} diff --git a/src/runtime/fork-state.c b/src/runtime/fork-state.c index a6d7dba7..8c8bc918 100644 --- a/src/runtime/fork-state.c +++ b/src/runtime/fork-state.c @@ -202,9 +202,9 @@ int fork_ipc_send_memory_regions(int ipc_sock, const guest_t *g, bool use_shm) #define MAX_USED_REGIONS 16 used_region_t used[MAX_USED_REGIONS]; unsigned int shim_sz = proc_get_shim_size(); - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire((guest_t *) (uintptr_t) g); int nregions = guest_get_used_regions(g, shim_sz, used, MAX_USED_REGIONS); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); uint32_t num_regions = (uint32_t) nregions; if (fork_ipc_write_all(ipc_sock, &num_regions, sizeof(num_regions)) < 0) @@ -751,6 +751,7 @@ int fork_ipc_send_process_state(int ipc_sock, const guest_region_t *regions_snapshot, uint32_t num_guest_regions, bool regions_tracker_stale_snapshot, + const uint64_t *dirty_blocks_snapshot, const guest_region_t *preannounced_snapshot, uint32_t num_preannounced) { @@ -803,6 +804,9 @@ int fork_ipc_send_process_state(int ipc_sock, fork_ipc_write_all(ipc_sock, regions_snapshot, num_guest_regions * sizeof(guest_region_t)) < 0) return -1; + if (fork_ipc_write_all(ipc_sock, dirty_blocks_snapshot, + GUEST_DIRTY_WORDS * sizeof(uint64_t)) < 0) + return -1; if (fork_ipc_write_all(ipc_sock, &num_preannounced, sizeof(num_preannounced)) < 0) @@ -1026,6 +1030,12 @@ int fork_ipc_recv_process_state(int ipc_fd, g->regions_tracker_stale = (regions_tracker_stale != 0) || (num_guest_regions > recv_regions); + if (fork_ipc_read_all(ipc_fd, g->dirty_blocks, + GUEST_DIRTY_WORDS * sizeof(uint64_t)) < 0) { + log_error("fork-child: failed to read dirty block bitmap"); + return -1; + } + uint32_t num_preannounced = 0; if (fork_ipc_read_all(ipc_fd, &num_preannounced, sizeof(num_preannounced)) < 0) { diff --git a/src/runtime/fork-state.h b/src/runtime/fork-state.h index 5533c968..7c4880be 100644 --- a/src/runtime/fork-state.h +++ b/src/runtime/fork-state.h @@ -19,7 +19,7 @@ /* Fork IPC protocol identity. Bump this whenever the header layout or ordered * fork payload changes incompatibly. */ -#define FORK_IPC_PROTOCOL_MAGIC 0x454C4651U /* "ELFQ" */ +#define FORK_IPC_PROTOCOL_MAGIC 0x454C4652U /* "ELFR" */ #define IPC_MAGIC_HEADER FORK_IPC_PROTOCOL_MAGIC #define IPC_MAGIC_SENTINEL 0x454C4F4BU /* "ELOK" */ @@ -150,6 +150,7 @@ int fork_ipc_send_process_state(int ipc_sock, const guest_region_t *regions_snapshot, uint32_t num_guest_regions, bool regions_tracker_stale_snapshot, + const uint64_t *dirty_blocks_snapshot, const guest_region_t *preannounced_snapshot, uint32_t num_preannounced); int fork_ipc_recv_process_state(int ipc_fd, diff --git a/src/runtime/forkipc.c b/src/runtime/forkipc.c index 72f77e0e..b792a85c 100644 --- a/src/runtime/forkipc.c +++ b/src/runtime/forkipc.c @@ -59,6 +59,7 @@ #include "utils.h" #include "core/shim-globals.h" +#include "core/mmap-fastpath.h" #include "runtime/forkipc.h" #include "runtime/fork-state.h" @@ -303,6 +304,7 @@ int fork_child_main(int ipc_fd, guest_destroy(&g); return 1; } + guest_rebuild_pte_present(&g); if (fork_ipc_recv_fd_table(ipc_fd, &g) < 0) { log_error("fork-child: failed to receive fd table"); @@ -539,6 +541,11 @@ int fork_child_main(int ipc_fd, */ shim_globals_rebuild_urandom_bitmap(); + if (!verbose) + mmap_fastpath_prepare_vcpu(&g, current_thread); + else + mmap_fastpath_disable(&g); + /* Now that current_thread is set, apply signal state. This must happen * after thread_register_main() so the per-thread blocked mask and altstack * are properly restored to the thread entry. @@ -610,7 +617,7 @@ typedef struct { vcpu_simd_state_t simd_state; } thread_create_args_t; -static void resolve_clone_stack_range(const guest_t *g, +static void resolve_clone_stack_range(guest_t *g, uint64_t child_stack, uint64_t *start_out, uint64_t *end_out) @@ -626,13 +633,10 @@ static void resolve_clone_stack_range(const guest_t *g, if (sp_off == 0 || sp_off > g->guest_size) return; - /* The region array is mutated under mmap_lock by any concurrent mmap or - * munmap, and clone does not otherwise take it. Reading it unlocked is a - * data race on g->regions and g->nregions, reported by ThreadSanitizer as - * soon as a sibling allocates while another thread clones. Neither caller - * holds a lock here, and mmap_lock is order 1, so taking it is safe. + /* The region array is mutated under mmap_lock. The acquire also drains EL1 + * mmap publications before clone resolves a newly allocated stack. */ - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); const guest_region_t *r = guest_region_find(g, sp_off - 1); if (r) { if (start_out) @@ -640,7 +644,7 @@ static void resolve_clone_stack_range(const guest_t *g, if (end_out) *end_out = r->end; } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); } /* Forward declaration: worker entry runs after sys_clone_thread */ @@ -1107,10 +1111,12 @@ static void *thread_create_and_run(void *arg) * how pthread_join works in musl: the joining thread does FUTEX_WAIT on * this address until it becomes 0. * - * Drain any deferred munmap of this thread's stack before waking the - * joiner: the parent may reuse the freed VA as soon as it returns from - * pthread_join, and reuse must not race with the deferred unmap. + * Drain any deferred munmap before publishing clear_child_tid. A joiner may + * observe the zero without ever sleeping in FUTEX_WAIT, then reuse the + * freed VA immediately; ordering only the wake after cleanup leaves a + * window where MAP_FIXED_NOREPLACE still sees the old stack VMA. */ + mem_cleanup_deferred_stack_unmaps(g, t); bool wake_ctid = false; if (t->clear_child_tid != 0) { uint32_t zero = 0; @@ -1125,7 +1131,6 @@ static void *thread_create_and_run(void *arg) (unsigned long long) t->clear_child_tid); } } - mem_cleanup_deferred_stack_unmaps(g, t); if (wake_ctid) futex_wake_one(g, t->clear_child_tid); @@ -1388,16 +1393,16 @@ static void *vm_clone_thread_run(void *arg) /* Set per-thread TLS pointer and enter worker run loop */ current_thread = t; thread_fork_barrier_check(); - log_debug("vm_clone tid=%lld starting on vCPU", (long long) thread_tid(t)); int wait_status = 0; int exit_code = vcpu_run_loop(vcpu, vexit, g, verbose, 0, &wait_status); - /* CLONE_CHILD_CLEARTID cleanup. Same ordering as thread_entry: drain - * deferred stack munmaps before waking the joiner so the parent does not - * reuse the VA before it is released. + /* CLONE_CHILD_CLEARTID cleanup. Same ordering as thread_entry: the zero + * itself, not just the futex wake, releases a joiner, so publish it only + * after the deferred stack mapping is gone. */ + mem_cleanup_deferred_stack_unmaps(g, t); bool wake_ctid = false; if (t->clear_child_tid != 0) { uint32_t zero = 0; @@ -1412,7 +1417,6 @@ static void *vm_clone_thread_run(void *arg) (unsigned long long) t->clear_child_tid); } } - mem_cleanup_deferred_stack_unmaps(g, t); if (wake_ctid) futex_wake_one(g, t->clear_child_tid); @@ -1534,6 +1538,17 @@ int64_t sys_clone(hv_vcpu_t vcpu, if ((flags & ~(uint64_t) 0xff) & LINUX_CLONE3_NS_FLAGS) return -LINUX_EINVAL; + /* Once an anonymous arena allocation becomes a live thread stack, munmap + * must pass through thread_collect_and_defer_stack_ranges(). Revoke arena + * generations before publishing the stack to the thread table so no EL1 + * fast munmap can bypass that lifetime rule. + */ + if (child_stack != 0) { + mmap_lock_acquire(g); + mmap_fastpath_revoke_all_locked(g, false); + mmap_lock_release(); + } + /* CLONE_THREAD: create a new thread in the same VM (not a new process) */ if (flags & LINUX_CLONE_THREAD) { return sys_clone_thread(vcpu, g, flags, child_stack, stack_map_start, @@ -1740,6 +1755,7 @@ int64_t sys_clone(hv_vcpu_t vcpu, mmap_fork_anon_shared_txn_t *anon_shared_txn = NULL; guest_region_t *regions_snapshot = NULL; + uint64_t *dirty_blocks_snapshot = NULL; guest_region_t preannounced_snapshot[GUEST_MAX_PREANNOUNCED]; int snapshot_shm_fd = -1; bool siblings_quiesced = false; @@ -1970,6 +1986,10 @@ int64_t sys_clone(hv_vcpu_t vcpu, } memcpy(regions_snapshot, g->regions, snap_sz); } + dirty_blocks_snapshot = malloc(sizeof(g->dirty_blocks)); + if (!dirty_blocks_snapshot) + goto fail_snapshot; + memcpy(dirty_blocks_snapshot, g->dirty_blocks, sizeof(g->dirty_blocks)); int npreannounced_snapshot = g->npreannounced; if (npreannounced_snapshot > 0) { memcpy(preannounced_snapshot, g->preannounced, @@ -1994,8 +2014,8 @@ int64_t sys_clone(hv_vcpu_t vcpu, uint32_t num_preannounced = (uint32_t) npreannounced_snapshot; if (fork_ipc_send_process_state( ipc_sock, regions_snapshot, num_guest_regions, - regions_tracker_stale_snapshot, preannounced_snapshot, - num_preannounced) < 0) { + regions_tracker_stale_snapshot, dirty_blocks_snapshot, + preannounced_snapshot, num_preannounced) < 0) { log_error("clone: failed to send process state"); goto fail_snapshot; } @@ -2076,6 +2096,7 @@ int64_t sys_clone(hv_vcpu_t vcpu, child_host_pid); free(regions_snapshot); + free(dirty_blocks_snapshot); if (snapshot_shm_fd >= 0) close(snapshot_shm_fd); return child_guest_pid; @@ -2083,6 +2104,7 @@ int64_t sys_clone(hv_vcpu_t vcpu, fail_snapshot: proc_cancel_child(child_guest_pid); free(regions_snapshot); + free(dirty_blocks_snapshot); if (snapshot_shm_fd >= 0) close(snapshot_shm_fd); diff --git a/src/runtime/futex.c b/src/runtime/futex.c index 8eed8e60..109cd3df 100644 --- a/src/runtime/futex.c +++ b/src/runtime/futex.c @@ -181,6 +181,24 @@ static inline bool futex_uaddr_is_aligned(uint64_t uaddr) return (uaddr & 0x3) == 0; } +static uint32_t *futex_word_nofault(const guest_t *g, + uint64_t uaddr, + int required_perms) +{ + uint64_t avail = 0; + uint32_t *word = guest_ptr_avail_nofault(g, uaddr, &avail, required_perms); + return word && avail >= sizeof(*word) ? word : NULL; +} + +/* Keep materialized futex words off mmap_lock. A miss is resolved before any + * bucket lock is held. + */ +static void futex_prefault_word(const guest_t *g, uint64_t uaddr) +{ + if (!futex_word_nofault(g, uaddr, MEM_PERM_R)) + guest_lazy_faultin(g, uaddr, sizeof(uint32_t)); +} + /* Unlink a waiter from its bucket's singly-linked list. Caller must hold * b->lock. Silently returns if the waiter is not in the list (already unlinked * by a wake/requeue). @@ -689,12 +707,14 @@ static int64_t futex_wait(guest_t *g, return -LINUX_EINVAL; } + futex_prefault_word(g, uaddr); + pthread_mutex_lock(&b->lock); /* Atomically read the futex word while holding the bucket lock. If it does * not match, return EAGAIN immediately. */ - uint32_t *word = (uint32_t *) guest_ptr(g, uaddr); + uint32_t *word = futex_word_nofault(g, uaddr, MEM_PERM_R); if (!word) { pthread_mutex_unlock(&b->lock); return -LINUX_EFAULT; @@ -930,6 +950,9 @@ static int64_t futex_requeue(guest_t *g, if (!futex_uaddr_is_aligned(uaddr) || !futex_uaddr_is_aligned(uaddr2)) return -LINUX_EINVAL; + if (do_cmp) + futex_prefault_word(g, uaddr); + unsigned idx_src = futex_hash(uaddr); unsigned idx_dst = futex_hash(uaddr2); futex_bucket_t *b_src = &buckets[idx_src]; @@ -948,7 +971,7 @@ static int64_t futex_requeue(guest_t *g, /* CMP_REQUEUE: atomically verify *uaddr == expected */ if (do_cmp) { - uint32_t *word = (uint32_t *) guest_ptr(g, uaddr); + uint32_t *word = futex_word_nofault(g, uaddr, MEM_PERM_R); if (!word) { if (idx_src != idx_dst) pthread_mutex_unlock(&b_dst->lock); @@ -1039,22 +1062,6 @@ static int64_t futex_wake_op(guest_t *g, if (!futex_uaddr_is_aligned(uaddr) || !futex_uaddr_is_aligned(uaddr2)) return -LINUX_EINVAL; - unsigned idx1 = futex_hash(uaddr); - unsigned idx2 = futex_hash(uaddr2); - futex_bucket_t *b1 = &buckets[idx1]; - futex_bucket_t *b2 = &buckets[idx2]; - - /* Lock ordering */ - if (idx1 == idx2) { - pthread_mutex_lock(&b1->lock); - } else if (idx1 < idx2) { - pthread_mutex_lock(&b1->lock); - pthread_mutex_lock(&b2->lock); - } else { - pthread_mutex_lock(&b2->lock); - pthread_mutex_lock(&b1->lock); - } - /* Decode operation and comparison from val3. Bits 31-28: operation (bit 31 * = OPARG_SHIFT flag, bits 30-28 = op) Bits 27-24: comparison operator Bits * 23-12: op_arg (operand for modify, 12-bit signed) Bits 11-0: cmp_arg @@ -1072,11 +1079,31 @@ static int64_t futex_wake_op(guest_t *g, /* FUTEX_OP_OPARG_SHIFT (bit 3 of wake_op): interpret op_arg as 1< 4 || wake_cmp > 5) + return -LINUX_ENOSYS; if (op_shift) op_arg = (int) (1U << (op_arg & 0x1F)); + futex_prefault_word(g, uaddr2); + + unsigned idx1 = futex_hash(uaddr); + unsigned idx2 = futex_hash(uaddr2); + futex_bucket_t *b1 = &buckets[idx1]; + futex_bucket_t *b2 = &buckets[idx2]; + + /* Lock ordering */ + if (idx1 == idx2) { + pthread_mutex_lock(&b1->lock); + } else if (idx1 < idx2) { + pthread_mutex_lock(&b1->lock); + pthread_mutex_lock(&b2->lock); + } else { + pthread_mutex_lock(&b2->lock); + pthread_mutex_lock(&b1->lock); + } + /* Atomically modify *uaddr2 */ - uint32_t *word2 = (uint32_t *) guest_ptr_w(g, uaddr2); + uint32_t *word2 = futex_word_nofault(g, uaddr2, MEM_PERM_W); if (!word2) { if (idx1 != idx2) pthread_mutex_unlock(&b2->lock); @@ -1837,6 +1864,8 @@ int64_t sys_futex_waitv(guest_t *g, */ if (!futex_uaddr_is_aligned(elts[i].uaddr)) return -LINUX_EINVAL; + + futex_prefault_word(g, elts[i].uaddr); } waitv_shared_t shared; @@ -1863,7 +1892,7 @@ int64_t sys_futex_waitv(guest_t *g, unsigned idx = futex_hash(uaddr); futex_bucket_t *b = &buckets[idx]; - uint32_t *word = (uint32_t *) guest_ptr(g, uaddr); + uint32_t *word = futex_word_nofault(g, uaddr, MEM_PERM_R); if (!word) { result_err = -LINUX_EFAULT; goto unlock_early; diff --git a/src/syscall/exec.c b/src/syscall/exec.c index 993edcd0..7685d874 100644 --- a/src/syscall/exec.c +++ b/src/syscall/exec.c @@ -817,23 +817,6 @@ static int64_t exec_handoff_to_leader(uint64_t path_gva, uint64_t envp_gva, const char *host_path) { - /* Release mmap_lock, which this thread's sc_execve wrapper holds, before - * touching the slot at all. - * - * Waiting for the slot under it deadlocks the second of two concurrent - * requesters against the leader: the leader takes the first request, - * releases exec_handoff_lock, and blocks acquiring mmap_lock to run it, - * while this thread holds mmap_lock waiting for a slot only that leader can - * free. Neither side is torn down, so no stop check breaks the cycle. - * - * Safe to drop this early because the handoff is the first thing sys_execve - * does for a non-leader, so nothing has yet read what mmap_lock protects, - * and the guest addresses published below are dereferenced by the leader - * under its own mmap_lock rather than here. Re-taken on every return path - * so the wrapper's unlock stays balanced. - */ - pthread_mutex_unlock(&mmap_lock); - pthread_mutex_lock(&exec_handoff_lock); /* Wait for the slot. A concurrent handoff either fails (freeing the slot) @@ -841,7 +824,6 @@ static int64_t exec_handoff_to_leader(uint64_t path_gva, */ if (!exec_handoff_wait_for(HANDOFF_EMPTY)) { pthread_mutex_unlock(&exec_handoff_lock); - pthread_mutex_lock(&mmap_lock); return -LINUX_EINTR; } @@ -860,7 +842,6 @@ static int64_t exec_handoff_to_leader(uint64_t path_gva, exec_handoff_set_state(HANDOFF_EMPTY); pthread_cond_broadcast(&exec_handoff_cond); pthread_mutex_unlock(&exec_handoff_lock); - pthread_mutex_lock(&mmap_lock); return -LINUX_ENAMETOOLONG; } exec_handoff.blocked_mask = @@ -889,7 +870,6 @@ static int64_t exec_handoff_to_leader(uint64_t path_gva, } pthread_mutex_unlock(&exec_handoff_lock); - pthread_mutex_lock(&mmap_lock); return result; } @@ -950,14 +930,8 @@ int64_t exec_run_handoff(hv_vcpu_t vcpu, guest_t *g, bool verbose) saved_mask = signal_save_blocked(); signal_set_blocked(adopt_mask); - /* sys_execve is written to run with mmap_lock held, which on the direct - * path its sc_execve wrapper takes. This path comes from the run loop, so - * take it here instead. - */ - pthread_mutex_lock(&mmap_lock); int64_t rc = sys_execve(vcpu, g, path_gva, argv_gva, envp_gva, verbose, host_path); - pthread_mutex_unlock(&mmap_lock); pthread_mutex_lock(&exec_handoff_lock); if (rc == SYSCALL_EXEC_HAPPENED) { @@ -1751,21 +1725,15 @@ int64_t sys_execve(hv_vcpu_t vcpu, * already have destroyed. It also has to precede the CLOEXEC sweep and * guest_reset: a sibling parked in read() on a fd about to close, or still * executing the old image's code, winds down against the memory and fd - * table its guest still expects. Both callers hold mmap_lock (order 1) - * across the whole syscall, and the teardown must not run under it: a - * sibling blocked in pthread_mutex_lock(&mmap_lock) inside sc_brk, sc_mmap, - * sc_munmap, sc_mprotect, or its own deferred stack unmap is reachable by - * none of the teardown wakes, so it can never reach a stop check and the - * join below would always time out. Measured before this release: four - * siblings looping on mmap/munmap took the fatal path every time. - * - * Dropping it here is safe because nothing in the teardown touches guest - * memory or the region table, and re-acquiring cannot contend: by the time - * it returns 0 no other guest thread is left to hold it. + * table its guest still expects. + */ + + /* Teardown must run without mmap_lock: a sibling blocked in an mmap-family + * syscall or its deferred stack unmap cannot reach a stop check while the + * lock is held. sys_execve acquires the lock below, immediately before the + * point of no return, after every sibling has stopped. */ - pthread_mutex_unlock(&mmap_lock); int survivors = thread_exec_de_thread(); - pthread_mutex_lock(&mmap_lock); /* The refusal above is a snapshot: a sibling could have created a CLONE_VM * child in the window between it and here. de_thread neither reaps nor @@ -1837,6 +1805,14 @@ int64_t sys_execve(hv_vcpu_t vcpu, return err; } + /* Input copying above may fault in argv/env strings from lazy anonymous + * mappings, so it must run without mmap_lock held. Serialize only after all + * recoverable validation is complete and immediately before replacing the + * guest address space. From this point every failure is fatal and both + * successful return paths release the lock explicitly. + */ + mmap_lock_acquire(g); + /* Point of no return. guest_reset() zeroes all guest memory. The old * process image is gone. All validation that can fail gracefully MUST * happen above this line. Failures below are unrecoverable; elfuse exits @@ -1996,6 +1972,7 @@ int64_t sys_execve(hv_vcpu_t vcpu, unlink(interp.resolved); exec_cleanup_inputs(argv, envp, argv_buf, envp_buf, path_host_buf, path_host_temp, interp_host_buf, interp_host_temp); + mmap_lock_release(); return SYSCALL_EXEC_HAPPENED; } @@ -2148,5 +2125,6 @@ int64_t sys_execve(hv_vcpu_t vcpu, exec_cleanup_inputs(argv, envp, argv_buf, envp_buf, path_host_buf, path_host_temp, interp_host_buf, interp_host_temp); + mmap_lock_release(); return SYSCALL_EXEC_HAPPENED; } diff --git a/src/syscall/internal.h b/src/syscall/internal.h index 774f415c..13b1b3c0 100644 --- a/src/syscall/internal.h +++ b/src/syscall/internal.h @@ -81,11 +81,10 @@ * under mmap_lock only by * exec_handoff_reset, and holds sig_lock * beneath it through - * signal_restore_blocked. A requester drops - * mmap_lock before taking it: waiting for - * the slot under mmap_lock deadlocks - * against the leader, which needs that lock - * to run the request that frees the slot + * signal_restore_blocked. sys_execve does + * not acquire mmap_lock until the point of + * no return, so a requester waiting for the + * slot never holds it * sig_lock (syscall/signal.c): signal handlers/pending/blocked * thread_lock (runtime/thread.c): thread table * sfd_lock (syscall/fd.c): special fd (never held with thread_lock) @@ -162,6 +161,19 @@ typedef int host_fd_t; extern pthread_mutex_t mmap_lock; /* Lock order: 1, mmap/brk + page tables */ extern pthread_mutex_t fd_lock; /* Lock order: 3, FD table */ +/* The only supported mmap_lock entry/exit API. Acquire drains the per-vCPU EL1 + * mmap rings before any caller can inspect semantic region state. + */ +void mmap_lock_acquire(guest_t *g); +void mmap_lock_release(void); +void mmap_lock_cond_wait(guest_t *g, pthread_cond_t *cond); + +/* Temporarily drop mmap_lock while retaining the host PT-gate reference, then + * reacquire without taking a second reference. Used only by lazy zeroing. + */ +void mmap_lock_drop_keep_gate(void); +void mmap_lock_reacquire_with_gate(guest_t *g); + /* FD table (defined in syscall/fdtable.c). */ extern fd_entry_t fd_table[FD_TABLE_SIZE]; diff --git a/src/syscall/mem.c b/src/syscall/mem.c index 68f41596..0da56cd9 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -19,11 +19,16 @@ #include #include #include +#include #include "debug/log.h" +#include "debug/syscall-hist.h" #include "utils.h" +#include "core/mmap-fastpath.h" + #include "proved/align.h" +#include "proved/mmap-fastpath.h" #include "runtime/thread.h" #include "syscall/linux-wire.h" @@ -37,6 +42,1049 @@ */ pthread_mutex_t mmap_lock = PTHREAD_MUTEX_INITIALIZER; /* Lock order: 1 */ +static pthread_once_t mmap_fastpath_env_once = PTHREAD_ONCE_INIT; +static bool mmap_fastpath_env_enabled; +static _Atomic bool mmap_fastpath_forced_off; + +static uint64_t find_free_gap_inner(const guest_t *g, + uint64_t length, + uint64_t min_addr, + uint64_t max_addr, + uint64_t align); +static bool mmap_fastpath_rewind_control_if_clean_locked( + guest_t *g, + shim_mmap_control_t *c); +static void mmap_fastpath_refill_thread_locked(guest_t *g, + thread_entry_t *t, + uint64_t request_len, + bool speculative); + +static void mmap_fastpath_read_env(void) +{ + const char *v = getenv("ELFUSE_MMAP_FASTPATH"); + mmap_fastpath_env_enabled = + !v || (strcmp(v, "0") != 0 && strcmp(v, "false") != 0); +} + +static bool mmap_fastpath_available(const guest_t *g) +{ + pthread_once(&mmap_fastpath_env_once, mmap_fastpath_read_env); + return mmap_fastpath_env_enabled && !g->is_rosetta && + !atomic_load_explicit(&mmap_fastpath_forced_off, + memory_order_acquire) && + !syscall_hist_enabled(); +} + +static shim_mmap_control_t *mmap_fastpath_control(const guest_t *g, int slot) +{ + if (!g || !g->host_base || slot < 0 || slot >= MAX_THREADS) + return NULL; + return (shim_mmap_control_t *) ((uint8_t *) g->host_base + + g->shim_data_base + SHIM_MMAP_CONTROL_BASE + + (uint64_t) slot * SHIM_MMAP_CONTROL_STRIDE); +} + + +static _Atomic uint32_t *mmap_fastpath_pt_gate(const guest_t *g) +{ + if (!g || !g->host_base) + return NULL; + return (_Atomic uint32_t *) ((uint8_t *) g->host_base + g->shim_data_base + + SHIM_MMAP_PT_GATE_OFF); +} + +/* mmap_lock serializes host writers. The gate extends that exclusion to EL1 + * fast munmap without making the per-vCPU producers contend with each other: + * after publishing gate=closed, wait for each producer's private active word. + */ +static void mmap_fastpath_host_gate_close(guest_t *g) +{ + _Atomic uint32_t *gate = mmap_fastpath_pt_gate(g); + if (!gate) + return; + uint32_t previous = + atomic_fetch_add_explicit(gate, 1, memory_order_acq_rel); + if (previous != 0) + return; + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + while (atomic_load_explicit(&c->retire.producer_active, + memory_order_acquire) != 0) + sched_yield(); + } +} + +static void mmap_fastpath_host_gate_open(guest_t *g) +{ + _Atomic uint32_t *gate = mmap_fastpath_pt_gate(g); + if (gate) { + uint32_t count = atomic_load_explicit(gate, memory_order_relaxed); + while (count != 0 && !atomic_compare_exchange_weak_explicit( + gate, &count, count - 1, memory_order_release, + memory_order_relaxed)) { + } + + /* exec resets the entire shim-data page while holding mmap_lock, + * including this implementation-only counter. Seeing zero here is + * therefore an already-open gate, not an underflow. + */ + } +} + +static _Thread_local guest_t *mmap_lock_guest; + +/* Record a length the guest actually obtained from this arena. Only real + * registrations enter the window: a request that missed and was served by the + * generic path never became arena traffic, and sizing the next arena for it + * would grow arenas for a workload that does not use them. + */ +static void mmap_fastpath_note_registration(shim_mmap_control_t *c, + uint64_t len) +{ + if (!len) + return; + c->publication_window[c->publication_seq & + (MMAP_FAST_PUBLICATION_WINDOW - 1)] = len; + c->publication_seq++; +} + +static void mmap_fastpath_window_reset(shim_mmap_control_t *c) +{ + for (unsigned i = 0; i < MMAP_FAST_PUBLICATION_WINDOW; i++) + c->publication_window[i] = 0; + c->publication_seq = 0; +} + +static void mmap_fastpath_disable_control(shim_mmap_control_t *c) +{ + uint32_t generation = + atomic_load_explicit(&c->generation, memory_order_relaxed) + 1; + if (generation == 0) + generation = 1; + atomic_store_explicit(&c->flags, 0, memory_order_relaxed); + atomic_store_explicit(&c->arena_base, 0, memory_order_relaxed); + atomic_store_explicit(&c->arena_limit, 0, memory_order_relaxed); + atomic_store_explicit(&c->cursor, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_start, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_end, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_generation, 0, memory_order_relaxed); + c->next_arena_size = MMAP_FAST_ARENA_MIN; + + /* Teardown (exec, fast-path disable) ends the workload the history + * described, so the next one starts from the minimum arena. + */ + mmap_fastpath_window_reset(c); + atomic_store_explicit(&c->pending_work, 0, memory_order_relaxed); + atomic_store_explicit(&c->generation, generation, memory_order_release); +} + +static void mmap_fastpath_drain_publications_locked(guest_t *g) +{ + if (!g || !g->host_base) + return; + + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + uint32_t head = atomic_load_explicit(&c->head, memory_order_relaxed); + uint32_t tail = atomic_load_explicit(&c->tail, memory_order_acquire); + if ((uint32_t) (tail - head) > SHIM_MMAP_RING_SIZE) { + log_fatal( + "mmap fast path: corrupt ring in vCPU slot %d " + "(head=%u tail=%u)", + slot, head, tail); + } + + uint64_t arena_base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t arena_limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + while (head != tail) { + const shim_mmap_entry_t *e = + &c->ring[head & (SHIM_MMAP_RING_SIZE - 1)]; + uint64_t addr = e->addr; + uint64_t len = e->len; + if ((addr & (GUEST_PAGE_SIZE - 1)) || !len || + (len & (GUEST_PAGE_SIZE - 1)) || addr < arena_base || + addr > arena_limit || len > arena_limit - addr || + e->prot != (LINUX_PROT_READ | LINUX_PROT_WRITE)) { + log_fatal( + "mmap fast path: invalid entry in vCPU slot %d " + "(addr=0x%llx len=0x%llx arena=0x%llx..0x%llx)", + slot, (unsigned long long) addr, (unsigned long long) len, + (unsigned long long) arena_base, + (unsigned long long) arena_limit); + } + if (guest_region_add_ex(g, addr, addr + len, + LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS | + LINUX_MAP_NORESERVE, + 0, NULL, -1) < 0) { + /* EL1 already returned this address to the guest. Continuing + * without semantic metadata would turn first touch into a false + * SIGSEGV, so fail closed on the violated provisioning + * invariant instead of silently corrupting process state. + */ + log_fatal( + "mmap fast path: region metadata exhausted while " + "draining vCPU slot %d", + slot); + } + mmap_fastpath_note_registration(c, len); + head++; + } + atomic_store_explicit(&c->head, head, memory_order_release); + } +} + +static void munmap_retire_commit_locked(guest_t *g, + const munmap_retire_entry_t *e, + uint64_t backing_start, + uint64_t backing_end) +{ + uint64_t start = e->addr; + uint64_t end = start + e->length; + + /* Publication drain ran first, so every mapping causally preceding this + * retirement is now represented in regions[]. A non-anonymous overlay in an + * arena indicates a missing revocation and must fail closed: EL1 has + * already invalidated the PTEs, so silently retaining such metadata would + * permit a later fault path to recreate them. + */ + for (int i = guest_region_first_end_above(g, start); i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->start >= end) + break; + if (r->end <= start) + continue; + if (!(r->flags & LINUX_MAP_ANONYMOUS) || + (r->flags & LINUX_MAP_SHARED) || r->backing_fd >= 0 || + r->overlay_active) { + log_fatal( + "munmap retire: non-fast mapping in arena " + "[0x%llx-0x%llx)", + (unsigned long long) start, (unsigned long long) end); + } + } + + guest_materialize_wait_range_locked(g, start, end); + + /* The PTE occupancy evidence was consumed by EL1, so use the conservative + * dirty bitmap to avoid touching huge never-materialized reservations. + * Retain the dirty bits and let a future lazy materialization zero only the + * backing that is actually reused. In particular, do not charge an + * unrelated VM exit (often the next mapping's first fault) with an eager + * memset of the retired range. guest_materialize_lazy_one() zeros dirty + * backing before publishing any new descriptor, so a future reader can + * never observe stale bytes. + * + * An earlier version of this function eagerly replaced the backing of large + * dirty runs (unmap + F_PUNCHHOLE + fresh mmap + remap) once the run's + * page-accurate materialized byte count crossed a size threshold, on the + * theory that handing the zero-fill to the host's demand-zero path beats a + * future software memset. Round-trip measurement (munmap immediately + * followed by a full re-touch of the same range, not just the munmap call + * in isolation) showed the opposite at every size tried from 16 MiB to 512 + * MiB: the replace path's own unmap/remap cost, paid synchronously while + * holding mmap_lock, exceeded the deferred memset it was meant to avoid, + * and grew faster than linearly with size. Retaining dirty bits + * unconditionally is cheaper in every case measured, so this function no + * longer special-cases large runs. + */ + + guest_region_remove(g, start, end); + if (backing_end > backing_start) + guest_retire_ptes_committed(g, backing_start, backing_end); + if (start < g->mmap_rw_gap_hint) + g->mmap_rw_gap_hint = start; + if (start < g->mmap_rx_gap_hint) + g->mmap_rx_gap_hint = start; +} + +void mmap_fastpath_drain_locked(guest_t *g) +{ + if (!g || !g->host_base) + return; + + /* Clear every advisory pending hint before acquire-snapshotting the rings. + * A publication racing this drain either appears in this snapshot or leaves + * its hint set for the next opportunistic drain. + * + * Acquire-snapshot every retirement tail before consuming any mmap + * publication. This is the cross-vCPU causal ordering required for "A mmap; + * publish pointer; B munmap": the acquire observes B's retire, then + * publication drain establishes A's semantic region before removal. + */ + uint32_t retire_tails[MAX_THREADS]; + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + atomic_exchange_explicit(&c->pending_work, 0, memory_order_acq_rel); + retire_tails[slot] = + atomic_load_explicit(&c->retire.tail, memory_order_acquire); + } + + mmap_fastpath_drain_publications_locked(g); + + bool retired_any = false; + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *producer = mmap_fastpath_control(g, slot); + uint32_t head = + atomic_load_explicit(&producer->retire.head, memory_order_relaxed); + uint32_t tail = retire_tails[slot]; + if ((uint32_t) (tail - head) > SHIM_MUNMAP_RETIRE_RING_SIZE) { + log_fatal( + "munmap retire: corrupt ring in vCPU slot %d " + "(head=%u tail=%u)", + slot, head, tail); + } + + while (head != tail) { + const munmap_retire_entry_t *e = + &producer->retire + .entries[head & (SHIM_MUNMAP_RETIRE_RING_SIZE - 1)]; + uint32_t arena_slot = + e->flags & SHIM_MUNMAP_RETIRE_F_ARENA_SLOT_MASK; + uint64_t charged_pages = + (e->flags & SHIM_MUNMAP_RETIRE_F_CHARGE_MASK) >> + SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT; + uint64_t charged_bytes = charged_pages * GUEST_PAGE_SIZE; + if (arena_slot >= MAX_THREADS || !e->length || + (e->addr & (GUEST_PAGE_SIZE - 1)) || + (e->length & (GUEST_PAGE_SIZE - 1)) || + e->addr > UINT64_MAX - e->length || charged_bytes > e->length) { + log_fatal("munmap retire: invalid entry in vCPU slot %d", slot); + } + + shim_mmap_control_t *arena = + mmap_fastpath_control(g, (int) arena_slot); + uint32_t generation = + atomic_load_explicit(&arena->generation, memory_order_acquire); + uint64_t base = + atomic_load_explicit(&arena->arena_base, memory_order_relaxed); + uint64_t cursor = + atomic_load_explicit(&arena->cursor, memory_order_relaxed); + uint64_t end = e->addr + e->length; + if (generation != e->arena_generation || e->addr < base || + end > cursor) { + log_fatal( + "munmap retire: stale arena generation/range " + "(producer=%d arena=%u gen=%u/%u)", + slot, arena_slot, e->arena_generation, generation); + } + + uint64_t backing_start = 0, backing_end = 0; + if (charged_bytes != 0) { + backing_start = atomic_load_explicit(&arena->materialized_start, + memory_order_relaxed); + backing_end = atomic_load_explicit(&arena->materialized_end, + memory_order_relaxed); + if (backing_start < e->addr) + backing_start = e->addr; + if (backing_end > end) + backing_end = end; + if (backing_end <= backing_start) + log_fatal( + "munmap retire: charged entry has no materialized " + "bounds (producer=%d arena=%u range=0x%llx..0x%llx " + "marker=0x%llx..0x%llx charged=0x%llx)", + slot, arena_slot, (unsigned long long) e->addr, + (unsigned long long) end, + (unsigned long long) atomic_load_explicit( + &arena->materialized_start, memory_order_relaxed), + (unsigned long long) atomic_load_explicit( + &arena->materialized_end, memory_order_relaxed), + (unsigned long long) charged_bytes); + } + + munmap_retire_commit_locked(g, e, backing_start, backing_end); + uint64_t consumed = atomic_load_explicit( + &producer->retire.consumed_bytes, memory_order_relaxed); + atomic_store_explicit(&producer->retire.consumed_bytes, + consumed + charged_bytes, + memory_order_release); + head++; + retired_any = true; + } + atomic_store_explicit(&producer->retire.head, head, + memory_order_release); + + /* The PT gate is closed while draining, so no producer can race this + * acknowledgement. Ring fullness remains the only hard per-vCPU + * backpressure; byte pressure is deliberately advisory. + */ + atomic_store_explicit(&producer->retire.cleanup_requested, 0, + memory_order_release); + } + + /* A stopped owner whose whole arena retired can collapse all published + * sub-extents back into its bump cursor. Like envelope reset, this must + * wait for the complete snapshot: overlapping retire records from sibling + * producers may otherwise observe a prematurely reset arena. + */ + if (current_thread && current_thread->sp_el1_slot >= 0) + mmap_fastpath_rewind_control_if_clean_locked( + g, mmap_fastpath_control(g, current_thread->sp_el1_slot)); + + /* EL1 may publish several charged retirements before this drain. Their PTEs + * are all already invalid, so clearing an arena's materialized envelope + * after the first commit would make later entries in the same snapshot lose + * their backing bounds. Restore the PTE-empty proof only after every + * snapshotted retirement has consumed the old envelope. + */ + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *arena = mmap_fastpath_control(g, slot); + if (!(atomic_load_explicit(&arena->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + continue; + uint64_t base = + atomic_load_explicit(&arena->arena_base, memory_order_relaxed); + uint64_t cursor = + atomic_load_explicit(&arena->cursor, memory_order_relaxed); + if (base < cursor && + guest_va_next_present_block(g, base, cursor) >= cursor) { + atomic_store_explicit(&arena->materialized_start, 0, + memory_order_relaxed); + atomic_store_explicit(&arena->materialized_end, 0, + memory_order_relaxed); + atomic_store_explicit(&arena->materialized_generation, 0, + memory_order_release); + } + } + + /* EL1 PTE stores cannot update guest_t's host-only cache generation. One + * bump per batch invalidates every host GVA translation cache after all + * retirement entries have committed. + */ + if (retired_any) + guest_pt_gen_bump(g); +} + +/* Top up the calling thread's arena before it runs dry. + * + * Refills otherwise happen only after the EL1 side has already missed: the fast + * path exhausts its arena, bails to HVC, and the host refills on the way + * through the slow path -- so the mapping that discovers the exhaustion always + * pays for it. Any host path that takes mmap_lock is already positioned to + * refill for free, having paid for the lock and the gate close. + * + * The water mark is the largest recent registration rather than a fixed + * fraction: what makes an arena useless is being unable to hold the mappings + * this workload actually makes, so an arena is spent exactly when its tail no + * longer covers one of them. That also bounds what the refill abandons -- + * relocating strands the unused tail until a later gap scan recovers it, and + * this way the strand is one typical mapping rather than a slice of an arena + * whose size nothing ties to the workload. A control with no history yet never + * triggers, which keeps startup from refilling on its first lock. + * + * Cheap enough for the lock path: an unavailable fast path or a thread with no + * slot costs one predictable branch, and a healthy arena costs three relaxed + * loads plus the window scan. Only the caller's own slot is considered; + * scanning all of them here would put a MAX_THREADS loop on every acquisition. + */ +static void mmap_fastpath_topup_locked(guest_t *g) +{ + if (!mmap_fastpath_available(g) || !current_thread || + current_thread->sp_el1_slot < 0) + return; + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + if (!c || !(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + return; + + uint64_t cursor = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t base = atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (cursor > limit) + return; + + uint64_t low_water = mmap_fastpath_window_max(c->publication_window); + if (!low_water || limit - cursor > low_water) + return; + + /* One control block describes one arena generation. Replacing it while a + * mapping from the used prefix is live strands that mapping: its later + * munmap cannot prove arena ownership and falls back to the host. This is + * especially expensive when one large mapping consumes the whole arena. The + * acquire above drained publications, so regions[] is authoritative. + */ + int first = guest_region_first_end_above(g, base); + if (first < g->nregions && g->regions[first].start < cursor) + return; + + mmap_fastpath_refill_thread_locked(g, current_thread, 0, true); +} + +static void mmap_lock_acquire_common(guest_t *g) +{ + pthread_mutex_lock(&mmap_lock); + mmap_fastpath_host_gate_close(g); + mmap_lock_guest = g; + mmap_fastpath_drain_locked(g); +} + +void mmap_lock_acquire(guest_t *g) +{ + mmap_lock_acquire_common(g); + mmap_fastpath_topup_locked(g); +} + +static void mmap_lock_acquire_for_fork(guest_t *g) +{ + /* Fork is about to revoke every arena. Keep the drain-before-region-read + * invariant without allocating a replacement arena that cannot survive this + * critical section. + */ + mmap_lock_acquire_common(g); +} + +void mmap_lock_release(void) +{ + mmap_fastpath_host_gate_open(mmap_lock_guest); + mmap_lock_guest = NULL; + pthread_mutex_unlock(&mmap_lock); +} + +void mmap_lock_cond_wait(guest_t *g, pthread_cond_t *cond) +{ + mmap_fastpath_host_gate_open(g); + mmap_lock_guest = NULL; + pthread_cond_wait(cond, &mmap_lock); + + /* pthread_cond_wait reacquires mmap_lock directly, so preserve the + * drain-before-region-read invariant of mmap_lock_acquire(). + */ + mmap_fastpath_host_gate_close(g); + mmap_lock_guest = g; + mmap_fastpath_drain_locked(g); +} + +void mmap_lock_drop_keep_gate(void) +{ + /* Dirty lazy-materialization drops mmap_lock around a potentially large + * memset. Retain this thread's gate reference so EL1 cannot retire the + * invalid PTE window and let the materializer recreate it afterwards. + * Another host thread may temporarily acquire mmap_lock; the refcounted + * gate remains closed until this owner finishes the materialization. + */ + mmap_lock_guest = NULL; + pthread_mutex_unlock(&mmap_lock); +} + +void mmap_lock_reacquire_with_gate(guest_t *g) +{ + pthread_mutex_lock(&mmap_lock); + mmap_lock_guest = g; + + /* EL1 mmap publication does not need the PT gate and may have progressed + * during the memset, so refresh semantic metadata before resuming. + */ + mmap_fastpath_drain_locked(g); +} + +static bool mmap_fastpath_has_pending_work(const guest_t *g) +{ + if (!g || !g->host_base) + return false; + + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + if (atomic_load_explicit(&c->pending_work, memory_order_acquire)) + return true; + } + return false; +} + +void mmap_fastpath_drain_vmexit(guest_t *g, bool fork_family_pending) +{ + if (!mmap_fastpath_has_pending_work(g)) + return; + + if (fork_family_pending) + mmap_lock_acquire_for_fork(g); + else + mmap_lock_acquire(g); + mmap_lock_release(); +} + +bool mmap_fastpath_current_producer_active(const guest_t *g) +{ + if (!g || !current_thread || current_thread->sp_el1_slot < 0) + return false; + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + return c && atomic_load_explicit(&c->retire.producer_active, + memory_order_acquire) != 0; +} + +void mmap_fastpath_note_materialized_locked(guest_t *g, + uint64_t start, + uint64_t end) +{ + if (!g || end <= start) + return; + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + if (!(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + continue; + uint64_t base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (start >= limit || end <= base) + continue; + uint32_t generation = + atomic_load_explicit(&c->generation, memory_order_relaxed); + uint64_t lo = start > base ? start : base; + uint64_t hi = end < limit ? end : limit; + uint32_t materialized = atomic_load_explicit( + &c->materialized_generation, memory_order_relaxed); + if (materialized == generation) { + uint64_t old_lo = atomic_load_explicit(&c->materialized_start, + memory_order_relaxed); + uint64_t old_hi = atomic_load_explicit(&c->materialized_end, + memory_order_relaxed); + if (old_lo < lo) + lo = old_lo; + if (old_hi > hi) + hi = old_hi; + } + atomic_store_explicit(&c->materialized_start, lo, memory_order_relaxed); + atomic_store_explicit(&c->materialized_end, hi, memory_order_relaxed); + atomic_store_explicit(&c->materialized_generation, generation, + memory_order_release); + + /* Fast-path arenas are allocated from disjoint VA ranges. Once this + * block has updated its owner, no later vCPU control can overlap it; + * avoid another 63 control-page probes on the single-vCPU hot path. + */ + break; + } +} + +/* mmap_fastpath_request_fits, mmap_fastpath_pow2_clamped, + * mmap_fastpath_window_max, and mmap_fastpath_arena_size live in + * proved/mmap-fastpath.h (included above): they are pure arithmetic over + * scalars, unlike everything else in this file, so make verify-mmapfastpath + * proves them directly instead of leaving them reviewed-by-eye. + */ + +/* speculative: refill because the arena is nearly spent, not because a request + * failed to fit. The fits check below would otherwise abandon such a call + * immediately -- a zero-length request fits any arena with a byte to spare. + */ +static void mmap_fastpath_refill_thread_locked(guest_t *g, + thread_entry_t *t, + uint64_t request_len, + bool speculative) +{ + if (!t || t->sp_el1_slot < 0) + return; + shim_mmap_control_t *c = mmap_fastpath_control(g, t->sp_el1_slot); + if (!c) + return; + if (!mmap_fastpath_available(g)) { + mmap_fastpath_disable_control(c); + return; + } + + /* Giant requests are deliberately slow-path-only. Do not abandon a useful + * small-request arena or poison its adaptive history. + */ + if (request_len > MMAP_FAST_ARENA_MAX) + return; + + uint64_t cursor = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t arena_base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + uint32_t flags = atomic_load_explicit(&c->flags, memory_order_relaxed); + uint64_t arena_size = mmap_fastpath_arena_size( + mmap_fastpath_window_max(c->publication_window), request_len); + if (!speculative && (flags & SHIM_MMAP_CTRL_ENABLED) && + mmap_fastpath_request_fits(cursor, limit, request_len, BLOCK_2MIB)) { + /* A capacity miss enters HVC, whose drain can rewind a completely + * retired arena before this refill check. Retaining that undersized + * arena merely because one more request fits makes the same miss recur + * every few operations and turns mmap latency into a periodic sawtooth. + * Grow an empty arena to the adaptive target; never relocate one that + * still contains allocations served by the current generation. + */ + bool empty = cursor == arena_base; + bool target_sized = + arena_base <= limit && limit - arena_base >= arena_size; + if (!empty || target_sized) + return; + } + + /* The owner is parked in HVC. Make the stranded tail immediately recyclable + * before the gap scan; mappings already served from the prefix were drained + * into regions[] on mmap_lock acquisition. + */ + if (flags & SHIM_MMAP_CTRL_ENABLED) + atomic_store_explicit(&c->cursor, limit, memory_order_relaxed); + + /* Prefer a real hole below the current high-water mark. Active sibling + * arena tails are excluded by mmap_fastpath_skip_reserved inside the gap + * allocator. Only grow mmap_next when no recyclable hole fits. + */ + uint64_t high = g->mmap_next; + if (high > g->mmap_limit) + high = g->mmap_limit; + uint64_t base = UINT64_MAX; + bool recycled = false; + if (high > MMAP_BASE) { + base = find_free_gap_inner(g, arena_size, MMAP_BASE, high, BLOCK_2MIB); + recycled = base != UINT64_MAX; + } + + if (!recycled) { + if (g->mmap_next > UINT64_MAX - (BLOCK_2MIB - 1)) { + mmap_fastpath_disable_control(c); + return; + } + base = ALIGN_UP(g->mmap_next, BLOCK_2MIB); + if (base > g->mmap_limit || arena_size > g->mmap_limit - base) { + mmap_fastpath_disable_control(c); + return; + } + } + uint64_t new_limit = base + arena_size; + + /* Carve VA only. Clearing stale descriptors once here makes every later + * bump allocation PTE-free without putting page-table work in EL1. Fresh + * bump-tail arenas beyond mmap_end cannot contain stale descriptors. + */ + if (recycled || base < g->mmap_end) { + if (guest_invalidate_ptes(g, base, new_limit) < 0) { + mmap_fastpath_disable_control(c); + return; + } + } + if (!recycled) { + g->mmap_next = new_limit; + if (g->mmap_rw_gap_hint < new_limit) + g->mmap_rw_gap_hint = new_limit; + } + + uint32_t generation = + atomic_load_explicit(&c->generation, memory_order_relaxed) + 1; + if (generation == 0) + generation = 1; + atomic_store_explicit(&c->arena_base, base, memory_order_relaxed); + atomic_store_explicit(&c->arena_limit, new_limit, memory_order_relaxed); + atomic_store_explicit(&c->cursor, base, memory_order_relaxed); + atomic_store_explicit(&c->materialized_start, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_end, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_generation, 0, memory_order_relaxed); + c->next_arena_size = arena_size; + + /* The window deliberately survives the generation change: sizing the next + * arena from the traffic that filled the previous one is the whole point, + * and clearing it here would restart the history at every refill. + */ + c->refill_count++; + if (recycled) + c->recycle_count++; + if (arena_size > c->peak_arena_size) + c->peak_arena_size = arena_size; + uint32_t control_flags = SHIM_MMAP_CTRL_ENABLED; + if (g_tlbi_range_supported) + control_flags |= SHIM_MMAP_CTRL_TLBIRANGE; + atomic_store_explicit(&c->flags, control_flags, memory_order_relaxed); + + /* This vCPU is stopped in HVC (or has never run), so host may acknowledge + * the freshly published descriptor on its behalf. Revocation deliberately + * does not do this, making an in-flight stale generation bail once. + */ + atomic_store_explicit(&c->consumer_generation, generation, + memory_order_relaxed); + atomic_store_explicit(&c->generation, generation, memory_order_release); +} + +void mmap_fastpath_refill_current_locked(guest_t *g, uint64_t request_len) +{ + mmap_fastpath_refill_thread_locked(g, current_thread, request_len, false); +} + +bool mmap_fastpath_allocate_current_locked(guest_t *g, + uint64_t request_len, + uint64_t *addr_out) +{ + if (!addr_out || !request_len || !mmap_fastpath_available(g) || + !current_thread || current_thread->sp_el1_slot < 0) + return false; + + mmap_fastpath_refill_thread_locked(g, current_thread, request_len, false); + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + if (!c || !(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + return false; + + uint64_t start = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (request_len >= BLOCK_2MIB) { + if (start > UINT64_MAX - (BLOCK_2MIB - 1)) + return false; + start = ALIGN_UP(start, BLOCK_2MIB); + } + if (start > limit || request_len > limit - start) + return false; + uint64_t end = start + request_len; + + if (guest_region_add_ex( + g, start, end, LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS | LINUX_MAP_NORESERVE, 0, + NULL, -1) < 0) + return false; + + atomic_store_explicit(&c->cursor, end, memory_order_release); + mmap_fastpath_note_registration(c, request_len); + if (end > g->mmap_end) + g->mmap_end = end; + *addr_out = start; + return true; +} + +/* Service mmap publication-ring backpressure without turning it into a global + * PT-gate rendezvous. The calling vCPU is stopped in HVC, so the host may + * advance only that vCPU's existing bump cursor after draining its prior + * publications. Sibling EL1 allocators own disjoint arenas and may continue + * publishing concurrently. + * + * This path deliberately refuses every operation that could require a host + * writer transaction: a pending retirement, a closed gate, arena refill or + * generation change all fall back to mmap_lock_acquire(). The acquire snapshots + * of every retire tail preserve the usual drain-before-metadata ordering for + * causally prior munmaps. + */ +bool mmap_fastpath_allocate_current_publication_only(guest_t *g, + uint64_t request_len, + uint64_t *addr_out) +{ + if (!g || !addr_out || !request_len || !mmap_fastpath_available(g) || + !current_thread || current_thread->sp_el1_slot < 0) + return false; + + pthread_mutex_lock(&mmap_lock); + _Atomic uint32_t *gate = mmap_fastpath_pt_gate(g); + if (!gate || atomic_load_explicit(gate, memory_order_acquire) != 0) + goto miss; + + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *producer = mmap_fastpath_control(g, slot); + uint32_t head = + atomic_load_explicit(&producer->retire.head, memory_order_relaxed); + uint32_t tail = + atomic_load_explicit(&producer->retire.tail, memory_order_acquire); + if (head != tail) + goto miss; + } + + mmap_fastpath_drain_publications_locked(g); + + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + uint32_t generation = + atomic_load_explicit(&c->generation, memory_order_acquire); + if (!(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED) || + atomic_load_explicit(&c->consumer_generation, memory_order_relaxed) != + generation) + goto miss; + + uint64_t start = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t base = atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (start == base) { + uint64_t target = mmap_fastpath_arena_size( + mmap_fastpath_window_max(c->publication_window), request_len); + if (base > limit || limit - base < target) + goto miss; + } + if (request_len >= BLOCK_2MIB) { + if (start > UINT64_MAX - (BLOCK_2MIB - 1)) + goto miss; + start = ALIGN_UP(start, BLOCK_2MIB); + } + if (start > limit || request_len > limit - start) + goto miss; + uint64_t end = start + request_len; + + if (guest_region_add_ex( + g, start, end, LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS | LINUX_MAP_NORESERVE, 0, + NULL, -1) < 0) + goto miss; + + atomic_store_explicit(&c->cursor, end, memory_order_release); + mmap_fastpath_note_registration(c, request_len); + if (end > g->mmap_end) + g->mmap_end = end; + *addr_out = start; + pthread_mutex_unlock(&mmap_lock); + return true; + +miss: + pthread_mutex_unlock(&mmap_lock); + return false; +} + +void mmap_fastpath_release_current_hint_locked(guest_t *g, + uint64_t addr, + uint64_t length) +{ + if (!current_thread || current_thread->sp_el1_slot < 0 || !length || + addr > UINT64_MAX - length) + return; + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + if (!c || !(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + return; + uint64_t base = atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (base >= limit || addr >= limit || addr + length <= base) + return; + + /* The owner is stopped in the HVC that reached sys_mmap, so it cannot race + * this descriptor update. Revoke its bump tail and committed free extents: + * the explicit hint must take precedence over every EL1-reserved hole, and + * the post-syscall refill will provision a new non-overlapping arena. + */ + mmap_fastpath_disable_control(c); +} + +/* Reuse a fully released arena in place. mmap_lock acquisition has drained this + * vCPU's publication ring, and sys_munmap has removed the last semantic region + * before calling here. The PTE occupancy index is the final guard: an arena is + * rewound only when no live metadata and no valid descriptor remain. The owner + * is stopped in HVC, so resetting its private bump cursor cannot race EL1. + * Keeping base/limit/generation unchanged avoids a host refill on the next mmap + * -- especially important when one 32 GiB request consumes the entire + * maximum-sized arena. + */ +static bool mmap_fastpath_rewind_control_if_clean_locked(guest_t *g, + shim_mmap_control_t *c) +{ + if (!g || !c) + return false; + if (!(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + return false; + + uint64_t base = atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + uint64_t cursor = atomic_load_explicit(&c->cursor, memory_order_relaxed); + if (base >= limit || cursor <= base) + return false; + + for (int i = 0; i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->start >= limit) + break; + if (r->end > base) + return false; + } + if (guest_va_next_present_block(g, base, limit) < limit) + return false; + + /* The stopped owner can safely discard pending sub-extents because the + * whole arena is becoming one bump-allocatable extent again. + */ + atomic_store_explicit(&c->cursor, base, memory_order_relaxed); + atomic_store_explicit(&c->materialized_start, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_end, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_generation, 0, memory_order_relaxed); + c->recycle_count++; + return true; +} + +static void mmap_fastpath_rewind_current_if_clean_locked(guest_t *g) +{ + if (!g || !current_thread || current_thread->sp_el1_slot < 0) + return; + mmap_fastpath_rewind_control_if_clean_locked( + g, mmap_fastpath_control(g, current_thread->sp_el1_slot)); +} + +void mmap_fastpath_prepare_vcpu(guest_t *g, thread_entry_t *t) +{ + mmap_lock_acquire(g); + mmap_fastpath_refill_thread_locked(g, t, 0, false); + mmap_lock_release(); +} + +void mmap_fastpath_revoke_all_locked(guest_t *g, bool shrink_high_water) +{ + mmap_fastpath_drain_locked(g); + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + if (c) + mmap_fastpath_disable_control(c); + } + + if (!shrink_high_water) + return; + uint64_t high = MMAP_BASE; + for (int i = 0; i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->start >= MMAP_BASE && r->start < g->mmap_limit && r->end > high) + high = r->end; + } + g->mmap_next = high; + if (g->mmap_rw_gap_hint > high) + g->mmap_rw_gap_hint = high; +} + +void mmap_fastpath_disable(guest_t *g) +{ + atomic_store_explicit(&mmap_fastpath_forced_off, true, + memory_order_release); + mmap_lock_acquire(g); + mmap_fastpath_revoke_all_locked(g, true); + mmap_lock_release(); +} + +void mmap_fastpath_skip_reserved(const guest_t *g, + uint64_t *start, + uint64_t length, + uint64_t align, + uint64_t max_addr) +{ + if (!g || !start || !length) + return; + bool advanced; + do { + advanced = false; + if (*start > max_addr || length > max_addr - *start) + return; + uint64_t end = *start + length; + for (int slot = 0; slot < MAX_THREADS; slot++) { + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + if (!(atomic_load_explicit(&c->flags, memory_order_acquire) & + SHIM_MMAP_CTRL_ENABLED)) + continue; + uint64_t cursor = + atomic_load_explicit(&c->cursor, memory_order_acquire); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (cursor < limit && *start < limit && end > cursor) { + *start = ALIGN_UP(limit, align); + advanced = true; + break; + } + } + } while (advanced); +} + /* Host kernel page size (16 KiB on Apple Silicon, typically 4 KiB on Intel * macOS). MAP_FIXED requires addr/length/offset multiples of this, so an * overlay onto a guest 4 KiB-aligned IPA is only applicable when the IPA @@ -111,6 +1159,11 @@ static int64_t sync_shared_aliases_range(guest_t *g, int backing_fd, uint64_t file_start, uint64_t file_end); +static int read_file_range_to_guest(guest_t *g, + uint64_t gpa, + int fd, + uint64_t file_off, + uint64_t len); static int region_count_after_removes(const guest_t *g, const remove_range_t *ranges, @@ -686,6 +1739,7 @@ static uint64_t find_free_gap_inner(const guest_t *g, uint64_t gap_start; if (!align_up_ok(min_addr, align, &gap_start)) return UINT64_MAX; + mmap_fastpath_skip_reserved(g, &gap_start, length, align, max_addr); /* Skip the prefix of regions entirely below gap_start in O(log n). After a * successful allocation the gap hint advances near or past the existing @@ -694,6 +1748,8 @@ static uint64_t find_free_gap_inner(const guest_t *g, */ for (int i = guest_region_first_end_above(g, gap_start); i < g->nregions; i++) { + mmap_fastpath_skip_reserved(g, &gap_start, length, align, max_addr); + /* A region can still slip below gap_start after the align_up_ok advance * below skips past a smaller adjacent region; keep the cheap guard. */ @@ -727,6 +1783,7 @@ static uint64_t find_free_gap_inner(const guest_t *g, } /* Check trailing space after all regions */ + mmap_fastpath_skip_reserved(g, &gap_start, length, align, max_addr); if (window_fits(gap_start, length, max_addr)) return gap_start; return UINT64_MAX; /* No suitable gap found */ @@ -1136,6 +2193,7 @@ static int64_t sys_mmap_high_va(guest_t *g, if (!host) goto fail; memset(host, 0, BLOCK_2MIB); + guest_dirty_clear_zeroed_range(g, gpa, gpa + BLOCK_2MIB); /* Detect freshness BEFORE guest_map_va_range so the decision is not * confused by a prior high-VA mmap into the same 2 MiB block. A fresh @@ -1214,28 +2272,11 @@ static int64_t sys_mmap_high_va(guest_t *g, } else if (prot != LINUX_PROT_NONE) { memset(map_host, 0, length); replaced_bytes_dirty = replacing_existing; - uint8_t *dst = map_host; - size_t remaining = length; - off_t file_off = (off_t) offset; - while (remaining > 0) { - ssize_t nr = pread(host_backing_fd, dst, remaining, file_off); - if (nr < 0) { - if (errno == EINTR) - continue; - - /* Real host I/O failure (not EINTR); previously the loop broke - * without setting ret and the syscall returned a "successful" - * partially-zero mapping. - */ - ret = linux_errno(); - goto fail; - } - if (nr == 0) - break; - dst += nr; - remaining -= (size_t) nr; - file_off += nr; - } + uint64_t gpa_for_addr = backing_gpa_start + (addr - va_base); + ret = read_file_range_to_guest(g, gpa_for_addr, host_backing_fd, offset, + length); + if (ret < 0) + goto fail; } /* Install L3 PTEs for the actual mapped range. Fresh blocks were fully @@ -1403,11 +2444,39 @@ static int64_t sys_mmap_high_va(guest_t *g, return ret; } +/* Page-table high-water slot (mmap_rx_end / mmap_end) for a TTBR0 mmap offset, + * or NULL if @off is not in either mmap arena. + */ +static uint64_t *mmap_pt_end_for_off(guest_t *g, uint64_t off) +{ + if (off >= MMAP_RX_BASE && off < MMAP_BASE) + return &g->mmap_rx_end; + if (off >= MMAP_BASE) + return &g->mmap_end; + return NULL; +} + +/* Advance the allocation high-water mark (mmap_rx_next / mmap_next) that fork + * IPC state transfer replays, for whichever arena @off belongs to. + */ +static void mmap_bump_next(guest_t *g, uint64_t off, uint64_t end) +{ + if (off >= MMAP_RX_BASE && off < MMAP_BASE) { + if (end > g->mmap_rx_next) + g->mmap_rx_next = end; + } else if (off >= MMAP_BASE) { + if (end > g->mmap_next) + g->mmap_next = end; + } +} + static int mremap_extend_range(guest_t *g, uint64_t off, uint64_t size, int prot) { + uint64_t *pt_end = mmap_pt_end_for_off(g, off); + if (prot == LINUX_PROT_NONE) { guest_invalidate_ptes(g, off, off + size); return 0; @@ -1418,10 +2487,67 @@ static int mremap_extend_range(guest_t *g, uint64_t ext_end = ALIGN_UP(off + size, BLOCK_2MIB); if (ext_end > g->guest_size) ext_end = g->guest_size; - if (guest_extend_page_tables(g, ext_start, ext_end, page_perms) < 0) + size_t nblocks = pt_end ? (size_t) ((ext_end - ext_start) / BLOCK_2MIB) : 0; + bool *block_preexisting = NULL; + if (nblocks) { + block_preexisting = calloc(nblocks, sizeof(*block_preexisting)); + if (!block_preexisting) + return -1; + for (size_t i = 0; i < nblocks; i++) + block_preexisting[i] = + guest_va_block_mapped(g, ext_start + (uint64_t) i * BLOCK_2MIB); + } + if (guest_extend_page_tables(g, ext_start, ext_end, page_perms) < 0) { + free(block_preexisting); return -1; - guest_update_perms(g, off, off + size, page_perms); + } + uint64_t saved_pt_end = pt_end ? *pt_end : 0; + if (pt_end && ext_end > *pt_end) + *pt_end = ext_end; + + for (size_t i = 0; i < nblocks; i++) { + if (block_preexisting[i]) + continue; + uint64_t b = ext_start + (uint64_t) i * BLOCK_2MIB; + uint64_t bend = b + BLOCK_2MIB; + if (bend > ext_end) + bend = ext_end; + uint64_t keep_start = off > b ? off : b; + uint64_t keep_end = off + size < bend ? off + size : bend; + if (keep_start <= b && keep_end >= bend) + continue; + if (guest_split_block(g, b) < 0) + goto fail; + if (b < keep_start && guest_invalidate_ptes(g, b, keep_start) < 0) + goto fail; + if (keep_end < bend && guest_invalidate_ptes(g, keep_end, bend) < 0) + goto fail; + } + + if (guest_update_perms(g, off, off + size, page_perms) < 0) + goto fail; + free(block_preexisting); return 0; + + /* Roll back: re-invalidate every fresh block whole (idempotent -- the + * forward pass already cleared the non-kept subranges) plus the kept [off, + * off+size) span, returning the range to its pre-extend state. + */ +fail: + for (size_t i = 0; i < nblocks; i++) { + if (block_preexisting[i]) + continue; + uint64_t b = ext_start + (uint64_t) i * BLOCK_2MIB; + uint64_t bend = b + BLOCK_2MIB; + if (bend > ext_end) + bend = ext_end; + (void) guest_invalidate_ptes(g, b, bend); + } + (void) guest_invalidate_ptes(g, off, off + size); + if (pt_end) + *pt_end = saved_pt_end; + free(block_preexisting); + return -1; } static int hvf_apply_file_overlay(guest_t *g, @@ -1452,6 +2578,12 @@ static int read_file_range_to_guest(guest_t *g, uint8_t *dst = host_ptr_for_gpa(g, gpa); if (!dst) return -LINUX_EFAULT; + + /* A short read, EOF, or later error may still leave nonzero bytes in the + * destination. Mark before the first pread so every exit is conservative. + */ + if (len <= UINT64_MAX - gpa) + guest_dirty_mark_range(g, gpa, gpa + len); size_t remaining = len; while (remaining > 0) { @@ -2018,12 +3150,16 @@ static int rollback_fresh_mmap_allocation(guest_t *g, { if (overlay_installed) hvf_remove_file_overlay(g, overlay_ipa, overlay_len); - if (guest_invalidate_ptes(g, start, start + length) < 0) + uint64_t end = start + length; + uint64_t cur_mmap_end = g->mmap_end; + uint64_t cur_mmap_rx_end = g->mmap_rx_end; + if (guest_invalidate_ptes(g, start, end) < 0) return -LINUX_ENOMEM; g->mmap_next = saved_mmap_next; - g->mmap_end = saved_mmap_end; + g->mmap_end = cur_mmap_end > saved_mmap_end ? cur_mmap_end : saved_mmap_end; g->mmap_rx_next = saved_mmap_rx_next; - g->mmap_rx_end = saved_mmap_rx_end; + g->mmap_rx_end = cur_mmap_rx_end > saved_mmap_rx_end ? cur_mmap_rx_end + : saved_mmap_rx_end; g->mmap_rw_gap_hint = saved_rw_gap_hint; g->mmap_rx_gap_hint = saved_rx_gap_hint; return 0; @@ -2355,6 +3491,8 @@ static int hvf_apply_file_overlay(guest_t *g, return -LINUX_EINTR; /* Being reaped; abandon the overlay */ int err = hvf_apply_file_overlay_quiesced(g, ipa, len, fd, file_off); thread_resume_siblings(); + if (err == 0 && len <= UINT64_MAX - ipa) + guest_dirty_mark_range(g, ipa, ipa + len); return err; } @@ -2394,6 +3532,13 @@ static int hvf_remove_file_overlay_quiesced(guest_t *g, return err; } + /* Restoring shm-backed slab pages may reveal an older nonzero snapshot. The + * following munmap/MAP_FIXED path will clear the bit only after it has + * actually zeroed a complete 2 MiB block. + */ + if (len <= UINT64_MAX - ipa) + guest_dirty_mark_range(g, ipa, ipa + len); + for (int i = 0; i < nsegments; i++) { if (hv_vm_map((uint8_t *) g->host_base + segments[i].ipa, segments[i].ipa, segments[i].len, @@ -2752,6 +3897,20 @@ int64_t sys_mmap(guest_t *g, mmap_fresh_init(&fresh); int host_backing_fd = -1; + /* Anonymous mappings defer page-table creation and zeroing to first touch + * (guest fault or host-side access), like MAP_NORESERVE always has. This + * keeps mmap()/munmap() cost independent of length: a multi-GiB reservation + * costs neither an eager PTE walk nor a full-length memset, and + * never-touched blocks consume no page-table pool. PROT_NONE stays a pure + * reservation (faults deliver SIGSEGV, not materialization), and MAP_FIXED + * keeps the eager path because it must atomically replace live mappings. + * Shared anonymous memory stays eager unless the caller opted into + * MAP_NORESERVE (the historical lazy set), since deferred zeroing has never + * been exercised against the fork snapshot paths for it. + */ + bool is_lazy = is_anon && !is_prot_none && + ((flags & LINUX_MAP_SHARED) == 0 || is_noreserve); + /* Tracks whether hvf_apply_file_overlay has installed a host * MAP_FIXED|MAP_SHARED mapping that the failure paths must undo if later * steps (page tables, region tracking) fall over. Without this, a @@ -2776,13 +3935,25 @@ int64_t sys_mmap(guest_t *g, * free()'d (free(NULL) is a no-op) before return. */ bool replaced_regions_removed = false; + + /* Linux kernel rejects MAP_FIXED with non-page-aligned address (checked + * below); the flag itself is needed early because it gates the lazy path. + */ + bool is_fixed = + (flags & LINUX_MAP_FIXED) || (flags & LINUX_MAP_FIXED_NOREPLACE); + if (is_fixed) + is_lazy = false; int track_flags = ((flags & LINUX_MAP_SHARED) ? LINUX_MAP_SHARED : LINUX_MAP_PRIVATE); if (is_anon) track_flags |= LINUX_MAP_ANONYMOUS; - /* Preserve MAP_NORESERVE in region metadata before merge checks run. */ - if (is_noreserve) + /* Preserve MAP_NORESERVE in region metadata before merge checks run. The + * same bit doubles as the internal lazy marker: guest_region_add_ex derives + * the region's deferred-PTE flag from it, and it is not guest visible + * (/proc/self/maps prints only prot and shared/private). + */ + if (is_noreserve || is_lazy) track_flags |= LINUX_MAP_NORESERVE; /* The memory syscall layer handles all mmap variants. Aligned file-backed @@ -2817,9 +3988,17 @@ int64_t sys_mmap(guest_t *g, if (length == 0) return -LINUX_ENOMEM; + /* A non-fixed nonzero address is a strong Linux hint. If it lands in the + * current (stopped) vCPU's invisible arena tail, release that tail before + * gap finding so implementation-only VA preparation does not perturb the + * address the application observes. Sibling arenas stay immutable without + * quiesce; their disjoint high-water placement makes self-overlap the + * normal and important case (allocator hinting near its previous result). + */ + if (!is_fixed && addr != 0) + mmap_fastpath_release_current_hint_locked(g, addr, length); + /* Linux kernel rejects MAP_FIXED with non-page-aligned address */ - bool is_fixed = - (flags & LINUX_MAP_FIXED) || (flags & LINUX_MAP_FIXED_NOREPLACE); if (is_fixed && (addr & 4095)) return -LINUX_EINVAL; @@ -2828,6 +4007,14 @@ int64_t sys_mmap(guest_t *g, */ bool is_noreplace = (flags & LINUX_MAP_FIXED_NOREPLACE) != 0; + /* A fixed mapping may replace an address previously handed out by an EL1 + * arena with a file/shared/stack-like mapping. Revoke all descriptors + * before making that semantic transition so a later fast munmap cannot + * classify it from the stale arena bounds. + */ + if (is_fixed) + mmap_fastpath_revoke_all_locked(g, false); + uint64_t result_off; /* Result as offset (0-based) */ if (is_fixed) { /* Addresses above TASK_SIZE (bit 63 set or beyond user VA range) are @@ -2869,6 +4056,7 @@ int64_t sys_mmap(guest_t *g, uint64_t fix_end = off + length; if (guest_range_hits_infra(g, off, fix_end)) return -LINUX_EINVAL; + guest_materialize_wait_range_locked(g, off, fix_end); result_off = off; @@ -3171,9 +4359,7 @@ int64_t sys_mmap(guest_t *g, return -LINUX_ENOMEM; } /* High-water mark for fork IPC state transfer */ - uint64_t rx_hwm = result_off + length; - if (rx_hwm > g->mmap_rx_next) - g->mmap_rx_next = rx_hwm; + mmap_bump_next(g, result_off, result_off + length); } else { /* RW (or PROT_NONE, or PROT_READ): allocate from main mmap region. * Honor the address hint if provided and within bounds. Some @@ -3232,9 +4418,7 @@ int64_t sys_mmap(guest_t *g, return -LINUX_ENOMEM; } /* High-water mark for fork IPC state transfer */ - uint64_t rw_hwm = result_off + length; - if (rw_hwm > g->mmap_next) - g->mmap_next = rw_hwm; + mmap_bump_next(g, result_off, result_off + length); } if (!region_has_capacity_after_removes(g, NULL, 0, 1)) { mmap_fresh_dispose(&fresh); @@ -3259,7 +4443,7 @@ int64_t sys_mmap(guest_t *g, guest_invalidate_ptes(g, result_off, result_off + length); } - if (!is_prot_none && !is_fixed && !is_noreserve) { + if (!is_prot_none && !is_fixed && !is_lazy) { /* Extend page tables for this specific allocation range only. * guest_extend_page_tables skips already-mapped blocks, so calling it * on pre-mapped regions is a no-op. This avoids creating entries for @@ -3317,16 +4501,24 @@ int64_t sys_mmap(guest_t *g, g->mmap_end = ext_end; } - /* Zero the mapped region */ + /* Zero the mapped region. RX mappings cannot be dirtied through their + * published PTEs, so a complete-block zero can make them clean again. + * Other mappings currently use writable stage-1 entries and must stay + * conservatively dirty even if their requested Linux prot is read-only. + */ memset((uint8_t *) g->host_base + result_off, 0, length); + if (needs_exec && !(prot & LINUX_PROT_WRITE)) + guest_dirty_clear_zeroed_range(g, result_off, result_off + length); } - /* MAP_NORESERVE: invalidate any stale PTEs (like PROT_NONE path) but track - * the region for lazy materialization on first fault. Page table entries - * will be created by guest_materialize_lazy() when the guest first touches - * a page in this range. + /* Lazy (private anonymous, incl. MAP_NORESERVE): invalidate any stale PTEs + * (like the PROT_NONE path) but track the region for lazy materialization + * on first fault. Page table entries will be created by + * guest_materialize_lazy() when the guest first touches a page in this + * range, or by the host-access fault-in path when a syscall targets the + * range before the guest ever touches it. */ - if (is_noreserve && !is_fixed) { + if (is_lazy) { guest_invalidate_ptes(g, result_off, result_off + length); } @@ -3392,6 +4584,7 @@ int64_t sys_mmap(guest_t *g, overlay_ipa = result_off; overlay_len = nf_overlay_len; } else { + guest_dirty_mark_range(g, result_off, result_off + length); uint8_t *dst = (uint8_t *) g->host_base + result_off; size_t remaining = length; off_t file_off = offset; @@ -3578,6 +4771,11 @@ int64_t sys_mremap(guest_t *g, */ if (guest_range_hits_infra(g, old_off, old_off + old_size)) return -LINUX_EINVAL; + if (old_off < g->guest_size) + guest_materialize_wait_range_locked(g, old_off, + old_size > g->guest_size - old_off + ? g->guest_size + : old_off + old_size); /* Verify the whole source range is covered by one logical VMA. A fork-aware * growth can split that VMA at the inherited/private boundary, but no @@ -3628,10 +4826,13 @@ int64_t sys_mremap(guest_t *g, /* Zero the trimmed region on its real backing (high-VA tails live at * gpa_base, not host_base + tail_off). */ - memset(host_ptr_for_gpa(g, src_gpa_base + (tail_off - src_start)), 0, - tail_end - tail_off); + uint64_t tail_gpa = src_gpa_base + (tail_off - src_start); + if (guest_invalidate_ptes(g, tail_off, tail_end) < 0) + return finish_mremap(&source, -LINUX_ENOMEM); + memset(host_ptr_for_gpa(g, tail_gpa), 0, tail_end - tail_off); + guest_dirty_clear_zeroed_range(g, tail_gpa, + tail_gpa + (tail_end - tail_off)); guest_region_remove_reserved(g, tail_off, tail_end, tail_remove_fd); - guest_invalidate_ptes(g, tail_off, tail_end); if (tail_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = tail_off; if (tail_off < g->mmap_rx_gap_hint) @@ -3658,6 +4859,7 @@ int64_t sys_mremap(guest_t *g, */ if (guest_range_hits_infra(g, new_off, new_off + new_size)) return finish_mremap(&source, -LINUX_EINVAL); + guest_materialize_wait_range_locked(g, new_off, new_off + new_size); /* Linux rejects MREMAP_FIXED when old and new ranges overlap */ uint64_t old_end = old_off + old_size, new_end = new_off + new_size; @@ -3799,7 +5001,14 @@ int64_t sys_mremap(guest_t *g, } (void) restore_snapshot_overlays_in_place(g, move.dest_snaps, move.dest_nsnaps); + int pt_err = + restore_snapshot_page_tables(g, new_off, new_off + new_size, + move.dest_snaps, move.dest_nsnaps); + if (pt_err < 0) + restore_err = pt_err; mremap_move_dispose(&move); + if (restore_err < 0) + return finish_mremap(&source, restore_err); return finish_mremap(&source, -LINUX_ENOMEM); } @@ -3856,14 +5065,22 @@ int64_t sys_mremap(guest_t *g, memset((uint8_t *) g->host_base + new_off + old_size, 0, new_size - old_size); + if (move.track.prot == LINUX_PROT_NONE) + guest_dirty_clear_zeroed_range(g, new_off, new_off + new_size); + else + guest_dirty_mark_range(g, new_off, new_off + new_size); + /* Remove old mapping */ if (old_size > 0) { - memset(host_ptr_for_gpa(g, src_gpa_base + (old_off - src_start)), 0, - old_size); + uint64_t old_gpa = src_gpa_base + (old_off - src_start); + bool invalidated = + guest_invalidate_ptes(g, old_off, old_off + old_size) == 0; + memset(host_ptr_for_gpa(g, old_gpa), 0, old_size); + if (invalidated) + guest_dirty_clear_zeroed_range(g, old_gpa, old_gpa + old_size); guest_region_remove_reserved(g, old_off, old_off + old_size, move.source_remove_fd); move.source_remove_fd = -1; - guest_invalidate_ptes(g, old_off, old_off + old_size); if (old_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = old_off; if (old_off < g->mmap_rx_gap_hint) @@ -3942,6 +5159,9 @@ int64_t sys_mremap(guest_t *g, } memset((uint8_t *) g->host_base + grow_off, 0, grow_len); + if (!(track.prot & LINUX_PROT_WRITE)) + guest_dirty_clear_zeroed_range(g, grow_off, + grow_off + grow_len); /* Update region tracking: remove old, add extended */ guest_region_remove_reserved(g, old_off, old_off + old_size, @@ -3954,14 +5174,7 @@ int64_t sys_mremap(guest_t *g, mark_region_backing_ro(g, old_off, old_off + new_size); /* Update high-water marks */ - uint64_t hwm = old_off + new_size; - if (old_off >= MMAP_RX_BASE && old_off < MMAP_BASE) { - if (hwm > g->mmap_rx_next) - g->mmap_rx_next = hwm; - } else if (old_off >= MMAP_BASE) { - if (hwm > g->mmap_next) - g->mmap_next = hwm; - } + mmap_bump_next(g, old_off, old_off + new_size); return finish_mremap(&source, (int64_t) old_addr); } @@ -4080,14 +5293,22 @@ int64_t sys_mremap(guest_t *g, memset((uint8_t *) g->host_base + new_off + old_size, 0, new_size - old_size); + if (track.prot == LINUX_PROT_NONE) + guest_dirty_clear_zeroed_range(g, new_off, new_off + new_size); + else + guest_dirty_mark_range(g, new_off, new_off + new_size); + /* Remove old mapping. Any live source overlay was already torn down * before the destination range was touched. */ - memset(host_ptr_for_gpa(g, src_gpa_base + (old_off - src_start)), 0, - old_size); + uint64_t old_gpa = src_gpa_base + (old_off - src_start); + bool invalidated = + guest_invalidate_ptes(g, old_off, old_off + old_size) == 0; + memset(host_ptr_for_gpa(g, old_gpa), 0, old_size); + if (invalidated) + guest_dirty_clear_zeroed_range(g, old_gpa, old_gpa + old_size); guest_region_remove_reserved(g, old_off, old_off + old_size, source_remove_fd); - guest_invalidate_ptes(g, old_off, old_off + old_size); if (old_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = old_off; if (old_off < g->mmap_rx_gap_hint) @@ -4100,14 +5321,7 @@ int64_t sys_mremap(guest_t *g, mark_region_backing_ro(g, new_off, new_off + new_size); /* Update high-water marks */ - uint64_t hwm = new_off + new_size; - if (new_off >= MMAP_RX_BASE && new_off < MMAP_BASE) { - if (hwm > g->mmap_rx_next) - g->mmap_rx_next = hwm; - } else if (new_off >= MMAP_BASE) { - if (hwm > g->mmap_next) - g->mmap_next = hwm; - } + mmap_bump_next(g, new_off, new_off + new_size); return finish_mremap(&source, (int64_t) guest_ipa(g, new_off)); } @@ -4197,6 +5411,8 @@ int64_t sys_madvise(guest_t *g, uint64_t addr, uint64_t length, int advice) */ if (!madvise_range_mapped(g, off, length)) return -LINUX_ENOMEM; + if (in_primary) + guest_materialize_wait_range_locked(g, off, off + length); uint64_t end = off + length; for (int i = 0; i < g->nregions; i++) { @@ -4312,11 +5528,38 @@ static int compare_range_pair(const void *a, const void *b) return 0; } +/* Coalesced sub-ranges of a munmap that must be zeroed. Sized so that even a + * pathologically fragmented lazy mapping (alternating materialized and + * untouched blocks) rarely overflows; on overflow the remainder of the region + * overlap is zeroed wholesale, which is always correct (zeroing already-zero + * slab bytes), just slower. + */ +#define MUNMAP_ZERO_RANGES_MAX 128 + +typedef struct { + uint64_t lo, hi; +} zero_range_t; + +static void zero_range_push(zero_range_t *ranges, + int *n, + uint64_t lo, + uint64_t hi) +{ + if (lo >= hi) + return; + if (*n > 0 && ranges[*n - 1].hi == lo) { + ranges[*n - 1].hi = hi; + return; + } + ranges[(*n)++] = (zero_range_t) {lo, hi}; +} + static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) { /* Reject munmap targeting VM infrastructure regions. */ if (guest_range_hits_infra(g, unmap_off, end)) return -LINUX_EINVAL; + guest_materialize_wait_range_locked(g, unmap_off, end); /* An interior removal from a file-backed region needs a second owned fd for * the surviving right half. Reserve it before changing overlays, page @@ -4337,26 +5580,91 @@ static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) return cleanup_err; } - /* Invalidate PTEs first. This may need to split a 2MiB block which can fail - * if the page table pool is exhausted. Failing before region removal keeps - * metadata consistent. + /* Record which sub-ranges need zeroing BEFORE the PTE invalidation below + * destroys the evidence. A pure anonymous, private, non-overlaid region + * needs no eager zeroing at all: its dirty bitmap already reflects every + * write that ever touched it -- the same guarantee + * munmap_retire_commit_locked() relies on to skip zeroing entirely on the + * EL1 fast munmap path, gated by the identical + * anonymous/private/no-backing-fd/no-overlay check below -- and + * hvf_remove_file_overlay_quiesced() explicitly marks a restored overlay's + * backing dirty via guest_dirty_mark_range() before this function ever sees + * it. guest_materialize_lazy_one() zeros dirty backing before publishing + * any new descriptor, so deferring here costs nothing at reuse time either; + * it only stops paying up front to zero bytes a future mapping may never + * touch. + * + * Anything else -- an actual file-backed region, or one whose overlay + * cleanup above could not tear down -- keeps the eager policy below: eager + * regions are zeroed across the whole overlap, and lazy (deferred-PTE, + * MAP_NORESERVE) regions only need their materialized 2MiB blocks zeroed. + * This keeps that fallback's cost proportional to memory actually touched + * instead of to the mapping length. */ - if (guest_invalidate_ptes(g, unmap_off, end) < 0) { - if (remove_fd >= 0) - close(remove_fd); - return -LINUX_ENOMEM; - } + zero_range_t zr[MUNMAP_ZERO_RANGES_MAX]; + int nzr = 0; for (int i = 0; i < g->nregions; i++) { - guest_region_t *r = &g->regions[i]; + const guest_region_t *r = &g->regions[i]; if (r->start >= end) break; if (r->end <= unmap_off) continue; if (r->prot == LINUX_PROT_NONE) continue; + if ((r->flags & LINUX_MAP_ANONYMOUS) && + !(r->flags & LINUX_MAP_SHARED) && r->backing_fd < 0 && + !r->overlay_active) + continue; uint64_t zstart = (r->start > unmap_off) ? r->start : unmap_off; uint64_t zend = (r->end < end) ? r->end : end; - memset((uint8_t *) g->host_base + zstart, 0, zend - zstart); + if (!r->noreserve) { + if (nzr >= MUNMAP_ZERO_RANGES_MAX) { + /* Out of slots: widen the last range instead of dropping any + * span that must be zeroed. Everything between ranges lies + * inside [unmap_off, end) and is being unmapped, so zeroing the + * gap as well is harmless. + */ + zr[nzr - 1].hi = zend; + continue; + } + zero_range_push(zr, &nzr, zstart, zend); + continue; + } + for (uint64_t b = zstart & ~(BLOCK_2MIB - 1); b < zend;) { + if (!guest_va_block_mapped(g, b)) { + /* Jump through the PTE occupancy index to the next materialized + * block. A huge untouched reservation therefore does no work + * proportional to its virtual length. + */ + b = guest_va_next_present_block(g, b + BLOCK_2MIB, zend); + continue; + } + uint64_t lo = (b > zstart) ? b : zstart; + uint64_t hi = (b + BLOCK_2MIB < zend) ? b + BLOCK_2MIB : zend; + if (nzr >= MUNMAP_ZERO_RANGES_MAX) { + /* Out of slots: fold the remainder of this overlap into the + * last range and stop scanning blocks. + */ + zr[nzr - 1].hi = zend; + break; + } + zero_range_push(zr, &nzr, lo, hi); + b += BLOCK_2MIB; + } + } + + /* Invalidate PTEs first. This may need to split a 2MiB block which can fail + * if the page table pool is exhausted. Failing before region removal keeps + * metadata consistent. + */ + if (guest_invalidate_ptes(g, unmap_off, end) < 0) { + if (remove_fd >= 0) + close(remove_fd); + return -LINUX_ENOMEM; + } + for (int i = 0; i < nzr; i++) { + memset((uint8_t *) g->host_base + zr[i].lo, 0, zr[i].hi - zr[i].lo); + guest_dirty_clear_zeroed_range(g, zr[i].lo, zr[i].hi); } guest_region_remove_reserved(g, unmap_off, end, remove_fd); if (unmap_off < g->mmap_rw_gap_hint) @@ -4381,7 +5689,7 @@ void mem_cleanup_deferred_stack_unmaps(guest_t *g, thread_entry_t *t) if (nranges <= 0) return; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); for (int i = 0; i < nranges; i++) { int rc = munmap_guest_range(g, starts[i], ends[i]); if (rc < 0) { @@ -4394,7 +5702,7 @@ void mem_cleanup_deferred_stack_unmaps(guest_t *g, thread_entry_t *t) } thread_drop_deferred_stack_unmap(t, starts[i], ends[i]); } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); } /* sys_munmap. */ @@ -4469,6 +5777,7 @@ int64_t sys_munmap(guest_t *g, uint64_t addr, uint64_t length) thread_finish_deferred_stack_ranges(txns, nranges); } } + mmap_fastpath_rewind_current_if_clean_locked(g); return 0; } @@ -4496,6 +5805,12 @@ int64_t sys_mprotect(guest_t *g, uint64_t addr, uint64_t length, int prot) if (addr > UINT64_MAX - length) return -LINUX_EINVAL; + /* Permission and VMA-shape changes are slow-path boundaries. Retire any + * already-published unmaps, then invalidate arena generations before the + * metadata/PTE edit so EL1 cannot act on the old anonymous classification. + */ + mmap_fastpath_revoke_all_locked(g, false); + if (addr <= 0x0000FFFFFFFFFFFFULL) { if (addr >= g->guest_size) { uint64_t mprot_end = addr + length; @@ -4551,6 +5866,7 @@ int64_t sys_mprotect(guest_t *g, uint64_t addr, uint64_t length, int prot) */ if (guest_range_hits_infra(g, mprot_off, mprot_end)) return -LINUX_EINVAL; + guest_materialize_wait_range_locked(g, mprot_off, mprot_end); /* Same max_prot check as the high-VA branch above. */ if ((prot & LINUX_PROT_WRITE) && @@ -4568,6 +5884,19 @@ int64_t sys_mprotect(guest_t *g, uint64_t addr, uint64_t length, int prot) if (prot != LINUX_PROT_NONE) { int page_perms = prot_to_perms(prot); + + /* Materialize lazy blocks in the range at their region's + * current prot before the block-granular extend below. The + * extend stamps whole 2MiB blocks with page_perms; on an + * unmaterialized lazy region that would hand every neighbor + * page OUTSIDE [mprot_off, mprot_end) the sub-range's + * permissions (region says RW, PTE says R-only, host-side + * writes EFAULT). guest_materialize_lazy covers block-within- + * region at region prot, so after this the extend is a no-op + * for lazy regions and update_perms below adjusts only the + * requested range. + */ + guest_lazy_faultin_locked(g, mprot_off, mprot_end - mprot_off); if (guest_extend_page_tables(g, mprot_off, mprot_end, page_perms) < 0) return -LINUX_ENOMEM; @@ -4967,7 +6296,14 @@ int mmap_fork_prepare_anon_shared(guest_t *g, if (!txn) return -LINUX_ENOMEM; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire_for_fork(g); + + /* fork callers have quiesced siblings. Drain their last publications, + * revoke every descriptor, and trim never-consumed arena tails before the + * legacy [MMAP_BASE,mmap_next) snapshot range is computed. + */ + mmap_fastpath_revoke_all_locked(g, true); + guest_materialize_wait_all_locked(g); size_t hps = host_page_size_cached(); @@ -5112,7 +6448,7 @@ int mmap_fork_prepare_anon_shared(guest_t *g, for (int k = 0; k < n_regions; k++) close(dup_fds[k]); close(fd); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); *txn_out = txn; return -LINUX_ENOMEM; } @@ -5125,7 +6461,7 @@ int mmap_fork_prepare_anon_shared(guest_t *g, for (int k = 0; k < n_regions; k++) close(dup_fds[k]); close(fd); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); *txn_out = txn; return nsnaps; } @@ -5165,7 +6501,7 @@ int mmap_fork_prepare_anon_shared(guest_t *g, close(fd); } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); *txn_out = txn; return 0; } @@ -5184,7 +6520,7 @@ int mmap_fork_abort_anon_shared(guest_t *g, mmap_fork_anon_shared_txn_t *txn = *txn_ptr; int rc = 0; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); for (int i = txn->noverlays - 1; i >= 0; i--) { const fork_overlay_snapshot_t *ovl = &txn->overlays[i]; @@ -5254,7 +6590,7 @@ int mmap_fork_abort_anon_shared(guest_t *g, } } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); mmap_fork_dispose_anon_shared_txn(txn_ptr); return rc; } @@ -5268,7 +6604,7 @@ int mmap_fork_restore_overlays(guest_t *g, const uint64_t *parent_ovl_start, const uint64_t *parent_ovl_end) { - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); int rc = 0; for (int i = 0; i < g->nregions; i++) { @@ -5357,6 +6693,6 @@ int mmap_fork_restore_overlays(guest_t *g, } } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); return rc; } diff --git a/src/syscall/proc.c b/src/syscall/proc.c index 85ddd72c..eb55a8de 100644 --- a/src/syscall/proc.c +++ b/src/syscall/proc.c @@ -37,6 +37,7 @@ #include "utils.h" #include "core/shim-globals.h" +#include "core/mmap-fastpath.h" #include "core/vdso.h" #include "runtime/futex.h" @@ -80,6 +81,23 @@ static _Atomic bool rosetta_enabled = true; */ static _Atomic bool rosetta_active = false; +static bool vcpu_exit_is_fork_family_syscall(hv_vcpu_t vcpu, + const hv_vcpu_exit_t *vexit) +{ + if (vexit->reason != HV_EXIT_REASON_EXCEPTION) + return false; + + uint64_t syndrome = vexit->exception.syndrome; + uint32_t ec = (uint32_t) ((syndrome >> 26) & 0x3f); + uint16_t imm = (uint16_t) (syndrome & 0xffff); + if (ec != 0x16 || imm != 5) + return false; + + uint64_t nr = 0; + hv_vcpu_get_reg(vcpu, HV_REG_X8, &nr); + return nr == SYS_clone || nr == SYS_clone3; +} + /* Process table for tracking direct and adopted fork children. Start small so * lifecycle tests exercise growth deterministically; expand under pid_lock as * the fork family grows. No pointer into this array survives unlocking. @@ -3668,9 +3686,9 @@ static bool vcpu_handle_el0_fault(guest_t *g, uint32_t fsc_type = (fsc >> 2) & 0xF; if (fsc_type == 0x01) { uint64_t fault_off = far_addr - g->ipa_base; - pthread_mutex_lock(&mmap_lock); - int mat = guest_materialize_lazy(g, fault_off); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_acquire(g); + int mat = guest_materialize_lazy_fault(g, fault_off); + mmap_lock_release(); if (mat == 0) { /* Page materialized; the helpers inside guest_materialize_lazy * populated the per-vCPU TLBI accumulator with the range just @@ -3681,6 +3699,21 @@ static bool vcpu_handle_el0_fault(guest_t *g, * (negative) entries would re-fault on the retry, looping until the * entry self-evicts. */ + shim_globals_counter_inc(g, SHIM_COUNTER_FAULT_MATERIALIZE); + switch ((tlbi_kind_t) cpu_tlbi_req.kind) { + case TLBI_RANGE: + shim_globals_counter_inc(g, SHIM_COUNTER_FAULT_TLBI_VAE); + break; + case TLBI_RANGE_LARGE: + shim_globals_counter_inc(g, SHIM_COUNTER_FAULT_TLBI_RVAE); + break; + case TLBI_BROADCAST: + shim_globals_counter_inc(g, SHIM_COUNTER_FAULT_TLBI_BCAST); + break; + case TLBI_NONE: + default: + break; + } tlbi_request_emit_to_vcpu(vcpu); return true; } @@ -3727,9 +3760,9 @@ static bool vcpu_handle_el0_fault(guest_t *g, uint64_t live_avail = 0; void *live_pt = NULL; if (stale_plausible) { - pthread_mutex_lock(&mmap_lock); - live_pt = guest_ptr_avail(g, far_addr, &live_avail, want_perm); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_acquire(g); + live_pt = guest_ptr_avail_nofault(g, far_addr, &live_avail, want_perm); + mmap_lock_release(); } if (live_pt) { /* Bound per vCPU and per (page, faulting PC). A genuinely stuck entry @@ -3923,6 +3956,32 @@ int vcpu_run_loop_with_hooks(hv_vcpu_t vcpu, drain_external_guest_signal(); + /* An HVF return can land inside the EL1 fast path's producer window, + * with retire.producer_active published for this vCPU's own slot. Only + * the guest clears that word, and every host path that takes mmap_lock + * waits for it in mmap_fastpath_host_gate_close(). Waiting there would + * block this thread on a vCPU that cannot run until this very thread + * re-enters it. Resume before dispatching either a cancellation or an + * exception. The producer window is bounded and non-blocking, so it + * retires at once; pending host attention survives to the next exit. + */ + while ((vexit->reason == HV_EXIT_REASON_CANCELED || + vexit->reason == HV_EXIT_REASON_EXCEPTION) && + mmap_fastpath_current_producer_active(g)) { + HV_CHECK_CTX(hv_vcpu_run(vcpu), vcpu, g); + drain_external_guest_signal(); + } + + /* Every return from HVF is a natural retirement point. Drain before + * dispatching syscalls, page faults, MAP_FIXED, fork/exec, signals, or + * exit so no host path can consult pre-munmap region metadata and + * rematerialize an EL1-invalidated page. The helper also drains mmap + * publications before the acquire-snapshotted retire entries. + */ + if (!mmap_fastpath_current_producer_active(g)) + mmap_fastpath_drain_vmexit( + g, vcpu_exit_is_fork_family_syscall(vcpu, vexit)); + /* Main: disarm timeout */ if (is_main) alarm(0); diff --git a/src/syscall/signal.c b/src/syscall/signal.c index a45d73a3..c92c7ae6 100644 --- a/src/syscall/signal.c +++ b/src/syscall/signal.c @@ -60,12 +60,14 @@ #include "hvutil.h" #include "core/shim-globals.h" +#include "core/mmap-fastpath.h" #include "core/vdso.h" #include "runtime/thread.h" #include "syscall/linux-wire.h" -#include "syscall/fd.h" /* signalfd_notify */ +#include "syscall/fd.h" /* signalfd_notify */ +#include "syscall/internal.h" #include "syscall/proc.h" /* proc_get_pid, proc_get_uid, SYSCALL_EXEC_HAPPENED */ #include "proved/sigframe.h" #include "syscall/signal.h" @@ -1987,6 +1989,14 @@ int64_t signal_sigaltstack(guest_t *g, uint64_t ss_gva, uint64_t old_ss_gva) */ if (ss.ss_sp > UINT64_MAX - ss.ss_size) return -LINUX_EINVAL; + + /* Alternate stacks have the same lifetime sensitivity as clone + * stacks: once registered, their unmap must take the host path + * instead of being classified only as an anonymous arena range. + */ + mmap_lock_acquire(g); + mmap_fastpath_revoke_all_locked(g, false); + mmap_lock_release(); t->altstack_sp = ss.ss_sp; t->altstack_flags = 0; t->altstack_size = ss.ss_size; @@ -2441,6 +2451,32 @@ int signal_take_termination_wait_status(void) return status; } +/* Pre-fault the candidate signal-frame windows (current stack and altstack top) + * before sig_lock is taken. The frame write in deliver_signal_locked runs under + * sig_lock; letting it materialize lazy stack pages there would acquire + * mmap_lock in descending lock order. The pre-fault is advisory -- the write + * path still faults in as a backstop -- but it makes the under-lock engagement + * unreachable in practice. Reading the altstack fields without sig_lock is + * benign for the same reason. + */ +static void signal_prefault_frame(hv_vcpu_t vcpu, guest_t *g) +{ + /* Worst-case alignment slack the frame can cost, matching the static_assert + * above: a larger margin would exceed LINUX_MINSIGSTKSZ and silently skip + * prefaulting a minimum-sized altstack. + */ + uint64_t need = sizeof(linux_rt_sigframe_t) + SIGFRAME_ALIGN - 1; + uint64_t sp = 0; + hv_vcpu_get_sys_reg(vcpu, HV_SYS_REG_SP_EL0, &sp); + if (sp > need && sp <= g->guest_size) + guest_lazy_faultin(g, sp - need, need); + thread_entry_t *thr = current_thread; + if (thr && thr->altstack_sp != 0 && + !(thr->altstack_flags & LINUX_SS_DISABLE) && thr->altstack_size > need) + guest_lazy_faultin(g, thr->altstack_sp + thr->altstack_size - need, + need); +} + /* signal_deliver_one() consumed a signal the guest never observes, so the * caller should look at the next one. Distinct from the documented 0/1/-1 * contract of deliver_signal_locked() and never escapes signal_deliver(). @@ -2451,6 +2487,8 @@ static int signal_deliver_one(hv_vcpu_t vcpu, guest_t *g, int *exit_code); int signal_deliver(hv_vcpu_t vcpu, guest_t *g, int *exit_code) { + signal_prefault_frame(vcpu, g); + /* Callers invoke this once per syscall epilogue, so stopping at the first * signal that turns out to be discarded (SIG_IGN, or a SIG_DFL disposition * of ignore/stop/continue) would let a lower-numbered ignored signal mask a @@ -2548,6 +2586,7 @@ int signal_deliver_fault(hv_vcpu_t vcpu, guest_t *g, int signum, int *exit_code) * threads faulting on the same signal collapse into one bit so one fault is * lost. Deliver directly here, never touching sig_state.pending. */ + signal_prefault_frame(vcpu, g); pthread_mutex_lock(&sig_lock); /* Linux force_sig_info_to_task(): a forced synchronous fault cannot be diff --git a/src/syscall/syscall.c b/src/syscall/syscall.c index a84fa81c..5781b712 100644 --- a/src/syscall/syscall.c +++ b/src/syscall/syscall.c @@ -72,6 +72,7 @@ #include "syscall/time.h" #include "core/shim-globals.h" +#include "core/mmap-fastpath.h" #include "debug/syscall-hist.h" @@ -181,9 +182,9 @@ typedef int64_t (*syscall_handler_t)(guest_t *g, { \ (void) g; (void) x0; (void) x1; (void) x2; \ (void) x3; (void) x4; (void) x5; (void) verbose; \ - pthread_mutex_lock(&mmap_lock); \ + mmap_lock_acquire(g); \ int64_t r = (body); \ - pthread_mutex_unlock(&mmap_lock); \ + mmap_lock_release(); \ return r; \ } @@ -500,16 +501,16 @@ static void sc_sync_regions_inline(guest_t *g) * position) cannot make us skip an entry permanently. */ for (int i = 0;; i++) { - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); if (i >= g->nregions) { - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); break; } const guest_region_t *r = &g->regions[i]; int duped = -1; if (r->shared && r->backing_fd >= 0) duped = dup(r->backing_fd); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); if (duped < 0) continue; (void) fsync(duped); @@ -540,7 +541,7 @@ static int64_t sc_sync_impl(guest_t *g) } pthread_mutex_unlock(&fd_lock); - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); for (int i = 0; i < g->nregions && n < (int) cap; i++) { const guest_region_t *r = &g->regions[i]; if (!r->shared || r->backing_fd < 0) @@ -550,7 +551,7 @@ static int64_t sc_sync_impl(guest_t *g) continue; hosts[n++] = duped; } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); /* fsync each dup outside both locks so a slow disk does not stall * concurrent FD or memory operations on other threads. @@ -736,7 +737,7 @@ static int64_t sc_mincore(guest_t *g, * never early-returns on a hole. */ uint8_t chunk[512]; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); int ri = guest_region_first_end_above(g, addr); for (uint64_t done = 0; done < npages;) { uint64_t batch = npages - done; @@ -751,13 +752,20 @@ static int64_t sc_mincore(guest_t *g, if (!mapped) has_hole = true; } - if (guest_write(g, vec + done, chunk, batch) < 0) { - pthread_mutex_unlock(&mmap_lock); + + /* sc_mincore holds mmap_lock while regions[] is swept. Materialize a + * valid lazy output block through the locked entry point, then use a + * no-fault copy so an invalid vec returns EFAULT instead of trying to + * acquire mmap_lock recursively. + */ + (void) guest_lazy_faultin_locked(g, vec + done, batch); + if (guest_write_nofault(g, vec + done, chunk, batch) < 0) { + mmap_lock_release(); return -LINUX_EFAULT; } done += batch; } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); return has_hole ? -LINUX_ENOMEM : 0; } @@ -974,6 +982,19 @@ static int64_t sc_set_tid_address(guest_t *g, return proc_get_pid(); } +static uint64_t mmap_fastpath_eligible_length(uint64_t addr, + uint64_t length, + uint64_t prot, + uint64_t flags) +{ + if (addr != 0 || prot != (LINUX_PROT_READ | LINUX_PROT_WRITE) || + (flags & ~(uint64_t) LINUX_MAP_NORESERVE) != + (LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS) || + length == 0 || length > UINT64_MAX - (GUEST_PAGE_SIZE - 1)) + return 0; + return (length + GUEST_PAGE_SIZE - 1) & ~(GUEST_PAGE_SIZE - 1); +} + static int64_t sc_mmap(guest_t *g, uint64_t x0, uint64_t x1, @@ -983,9 +1004,23 @@ static int64_t sc_mmap(guest_t *g, uint64_t x5, bool verbose) { - pthread_mutex_lock(&mmap_lock); - int64_t r = sys_mmap(g, x0, x1, (int) x2, (int) x3, (int) x4, (int64_t) x5); - pthread_mutex_unlock(&mmap_lock); + uint64_t refill_len = mmap_fastpath_eligible_length(x0, x1, x2, x3); + uint64_t arena_addr = 0; + if (refill_len && mmap_fastpath_allocate_current_publication_only( + g, refill_len, &arena_addr)) + return (int64_t) arena_addr; + + mmap_lock_acquire(g); + int64_t r; + if (refill_len && + mmap_fastpath_allocate_current_locked(g, refill_len, &arena_addr)) { + r = (int64_t) arena_addr; + } else { + r = sys_mmap(g, x0, x1, (int) x2, (int) x3, (int) x4, (int64_t) x5); + if (r >= 0 && refill_len) + mmap_fastpath_refill_current_locked(g, refill_len); + } + mmap_lock_release(); log_debug(" mmap(0x%llx, 0x%llx) \xe2\x86\x92 0x%llx", (unsigned long long) x0, (unsigned long long) x1, (unsigned long long) (uint64_t) r); @@ -1002,9 +1037,10 @@ static int64_t sc_mremap(guest_t *g, bool verbose) { (void) x5; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); + mmap_fastpath_revoke_all_locked(g, false); int64_t r = sys_mremap(g, x0, x1, x2, (int) x3, x4); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); log_debug(" mremap(0x%llx, 0x%llx, 0x%llx, 0x%x) \xe2\x86\x92 0x%llx", (unsigned long long) x0, (unsigned long long) x1, (unsigned long long) x2, (int) x3, @@ -2189,10 +2225,7 @@ static int64_t sc_execve(guest_t *g, (void) x3; (void) x4; (void) x5; - pthread_mutex_lock(&mmap_lock); - int64_t r = sys_execve(current_thread->vcpu, g, x0, x1, x2, verbose, NULL); - pthread_mutex_unlock(&mmap_lock); - return r; + return sys_execve(current_thread->vcpu, g, x0, x1, x2, verbose, NULL); } static int64_t sc_execveat(guest_t *g, @@ -2208,8 +2241,9 @@ static int64_t sc_execveat(guest_t *g, hv_vcpu_t vcpu = current_thread->vcpu; int dirfd = (int) x0, flags = (int) x4; - /* Resolve the target path before taking mmap_lock (path resolution may call - * fd_to_host / openat which do not need mmap_lock). + /* Resolve the target path before entering the exec transaction. Path + * resolution may call fd_to_host / openat and does not need mmap_lock; + * sys_execve takes it at the point of no return. */ uint64_t path_gva = x1; char resolved[LINUX_PATH_MAX]; @@ -2272,7 +2306,6 @@ static int64_t sc_execveat(guest_t *g, need_resolve = true; } - pthread_mutex_lock(&mmap_lock); int64_t r; if (need_resolve) { /* Use the host-resolved path directly so execveat does not copy a host @@ -2282,7 +2315,6 @@ static int64_t sc_execveat(guest_t *g, } else { r = sys_execve(vcpu, g, path_gva, x2, x3, verbose, NULL); } - pthread_mutex_unlock(&mmap_lock); return r; } diff --git a/src/syscall/sysvipc.c b/src/syscall/sysvipc.c index 503b2510..4f71db4a 100644 --- a/src/syscall/sysvipc.c +++ b/src/syscall/sysvipc.c @@ -255,7 +255,12 @@ int64_t sys_shmat(guest_t *g, int shmid, uint64_t shmaddr_gva, int shmflg) return gva; /* propagate mmap error */ } - /* Copy host shm content into guest memory */ + /* Copy host shm content into guest memory. sys_shmat runs under mmap_lock + * (SC_LOCKED), so the resolve-time lazy fault-in inside guest_write would + * self-deadlock on it; materialize the fresh anonymous mapping through the + * locked variant first. + */ + guest_lazy_faultin_locked(g, (uint64_t) gva, seg_size); if (guest_write(g, (uint64_t) gva, host_addr, seg_size) < 0) { shmdt(host_addr); return -LINUX_EFAULT; @@ -314,7 +319,11 @@ int64_t sys_shmdt(guest_t *g, uint64_t shmaddr_gva) /* Write back guest modifications to host shm (unless read-only) */ if (!entry.rdonly) { - /* Read guest memory back to host shm buffer */ + /* Read guest memory back to host shm buffer. Same SC_LOCKED + * self-deadlock hazard as the shmat copy-in: pages the guest never + * touched may still be unmaterialized. + */ + guest_lazy_faultin_locked(g, entry.guest_gva, entry.size); guest_read(g, entry.guest_gva, entry.host_addr, entry.size); } diff --git a/tests/bench-mmap-fresh b/tests/bench-mmap-fresh new file mode 100755 index 00000000..11cc280d --- /dev/null +++ b/tests/bench-mmap-fresh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Accumulate auxiliary fresh bump-tail mmap timing across independent elfuse +# processes. One process cannot retain enough large fresh mappings to overcome +# CNTVCT's 41.7-ns tick without exhausting guest VA. Each invocation below gets +# a new guest, while its in-guest timer excludes process startup from the +# measurement. + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ELFUSE="${ELFUSE:-${ROOT_DIR}/build/elfuse}" +BENCH="${BENCH_MMAP_BIN:-${ROOT_DIR}/build/bench-mmap}" + +# 1024 ticks = about 42.7 us on Apple Silicon. This bounds the aggregate counter +# quantization to roughly 0.1%, while keeping the largest cases timely. +TARGET_TICKS="${BENCH_MMAP_FRESH_TARGET_TICKS:-1024}" +MAX_RUNS="${BENCH_MMAP_FRESH_MAX_RUNS:-8192}" + +# size in bytes : mappings per fresh guest run +CASES=( + "4096:1000" + "65536:1000" + "1048576:1000" + "2097152:1000" + "8388608:256" + "134217728:16" + "1073741824:8" + "8589934592:4" + "34359738368:2" +) + +# Optional positional cases use the same size:count spelling as CASES, e.g. +# tests/bench-mmap-fresh 1073741824:8 8589934592:4 +if [ "$#" -gt 0 ]; then + CASES=("$@") +fi + +if [ ! -x "$ELFUSE" ] || [ ! -x "$BENCH" ]; then + echo "build build/elfuse and build/bench-mmap first" >&2 + exit 2 +fi + +printf 'Auxiliary fresh bump-tail mmap: target %s aggregate CNTVCT ticks\n' \ + "$TARGET_TICKS" +printf '%-10s %8s %6s %14s\n' size count runs 'fresh mmap ns' + +for case in "${CASES[@]}"; do + size="${case%%:*}" + count="${case##*:}" + total_ticks=0 + total_ops=0 + runs=0 + ns_per_tick= + failed=0 + + while awk -v ticks="$total_ticks" -v target="$TARGET_TICKS" \ + 'BEGIN { exit !(ticks < target) }'; do + if [ "$runs" -ge "$MAX_RUNS" ]; then + echo "fresh benchmark exceeded ${MAX_RUNS} runs for ${size}" >&2 + failed=1 + break + fi + if ! output="$("$ELFUSE" "$BENCH" fresh "$size" "$count" 2>&1)"; then + echo "fresh guest run failed for size=${size}, count=${count}:" >&2 + echo "$output" >&2 + failed=1 + break + fi + raw_ticks="$(awk '{for (i = 1; i <= NF; i++) if ($i ~ /^ticks=/) {sub(/^ticks=/, "", $i); print $i}}' <<< "$output")" + read_ticks="$(awk '{for (i = 1; i <= NF; i++) if ($i ~ /^read_ticks=/) {sub(/^read_ticks=/, "", $i); print $i}}' <<< "$output")" + ns_per_tick="$(awk '{for (i = 1; i <= NF; i++) if ($i ~ /^ns_per_tick=/) {sub(/^ns_per_tick=/, "", $i); print $i}}' <<< "$output")" + if [ -z "$raw_ticks" ] || [ -z "$read_ticks" ] || [ -z "$ns_per_tick" ]; then + echo "unexpected benchmark output: $output" >&2 + failed=1 + break + fi + total_ticks="$(awk -v total="$total_ticks" -v raw="$raw_ticks" -v read="$read_ticks" 'BEGIN { print total + raw - read }')" + total_ops=$((total_ops + count)) + runs=$((runs + 1)) + done + + human="$(awk -v n="$size" 'BEGIN { if (n >= 1073741824) printf "%g GiB", n / 1073741824; else if (n >= 1048576) printf "%g MiB", n / 1048576; else printf "%g KiB", n / 1024 }')" + if [ "$failed" -ne 0 ]; then + printf '%-10s %8s %6s %14s\n' "$human" "$count" "$runs" FAILED + continue + fi + ns_per_op="$(awk -v ticks="$total_ticks" -v ops="$total_ops" -v ns="$ns_per_tick" 'BEGIN { printf "%.2f", ticks * ns / ops }')" + printf '%-10s %8s %6s %14s\n' "$human" "$count" "$runs" "$ns_per_op" +done diff --git a/tests/bench-mmap-isolated b/tests/bench-mmap-isolated new file mode 100755 index 00000000..761afaca --- /dev/null +++ b/tests/bench-mmap-isolated @@ -0,0 +1,256 @@ +#!/usr/bin/env bash +# mmap/munmap performance tests isolated across elfuse processes. +# +# Section A launches a brand new elfuse process for every timed mmap/munmap +# pair. Section B takes an outer median across fresh processes for each size. +# Section C launches one process per size because its large dirty sweeps are +# already distributions over individually timed operations. Section D pairs an +# empty VM exit with the next exit that drains EL1 fast-path work. + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ELFUSE="${ELFUSE:-${ROOT_DIR}/build/elfuse}" +BENCH="${BENCH_MMAP_BIN:-${ROOT_DIR}/build/bench-mmap}" + +# One timed call is comparable to CNTVCT's 41.7-ns period. Accumulate corrected +# ticks across fresh processes before converting them to a per-call result. +ONESHOT_TARGET_TICKS="${BENCH_MMAP_ONESHOT_TARGET_TICKS:-256}" +ONESHOT_MAX_RUNS="${BENCH_MMAP_ONESHOT_MAX_RUNS:-4096}" +MATERIALIZED_GUESTS="${BENCH_MMAP_MATERIALIZED_GUESTS:-15}" +HOST_DRAIN_SAMPLES="${BENCH_MMAP_HOST_DRAIN_SAMPLES:-31}" +SIZE_POINTS="${BENCH_MMAP_SIZE_POINTS:-41}" + +log_sizes() +{ + awk -v min="$1" -v max="$2" -v count="$3" 'BEGIN { + page = 4096 + last = 0 + span = log(max) - log(min) + for (i = 0; i < count; i++) { + if (i == 0) + value = min + else if (i == count - 1) + value = max + else { + exact = exp(log(min) + span * i / (count - 1)) + value = int((exact + page - 1) / page) * page + if (value <= last) + value = last + page + } + printf "%.0f\n", value + last = value + } + }' +} + +dirty_ops() +{ + local size="$1" + if [ "$size" -le 65536 ]; then + printf '2048\n' + elif [ "$size" -le 262144 ]; then + printf '1024\n' + elif [ "$size" -le 1048576 ]; then + printf '512\n' + elif [ "$size" -le 2097152 ]; then + printf '256\n' + elif [ "$size" -le 8388608 ]; then + printf '128\n' + elif [ "$size" -le 67108864 ]; then + printf '32\n' + elif [ "$size" -le 268435456 ]; then + printf '24\n' + else + printf '8\n' + fi +} + +if [ ! -x "$ELFUSE" ] || [ ! -x "$BENCH" ]; then + echo "build build/elfuse and build/bench-mmap first" >&2 + exit 2 +fi +if ! [[ "$MATERIALIZED_GUESTS" =~ ^[1-9][0-9]*$ ]]; then + echo "BENCH_MMAP_MATERIALIZED_GUESTS must be a positive integer" >&2 + exit 2 +fi +if ! [[ "$HOST_DRAIN_SAMPLES" =~ ^[1-9][0-9]*$ ]] || + [ "$HOST_DRAIN_SAMPLES" -gt 100000 ]; then + echo "BENCH_MMAP_HOST_DRAIN_SAMPLES must be in 1..100000" >&2 + exit 2 +fi +if ! [[ "$SIZE_POINTS" =~ ^[0-9]+$ ]] || [ "$SIZE_POINTS" -lt 35 ] || + [ "$SIZE_POINTS" -gt 45 ]; then + echo "BENCH_MMAP_SIZE_POINTS must be in 35..45" >&2 + exit 2 +fi + +# A and B span 4 KiB to 32 GiB. C retains its 4 KiB to 1 GiB bounds. +SIZES=() +while IFS= read -r size; do + SIZES+=("$size") +done < <(log_sizes 4096 34359738368 "$SIZE_POINTS") + +DIRTY_CASES=() +while IFS= read -r size; do + DIRTY_CASES+=("${size}:$(dirty_ops "$size")") +done < <(log_sizes 4096 1073741824 "$SIZE_POINTS") + +human() +{ + awk -v n="$1" 'BEGIN { + if (n >= 1073741824) printf "%g GiB", n / 1073741824 + else if (n >= 1048576) printf "%g MiB", n / 1048576 + else printf "%g KiB", n / 1024 + }' +} + +# field Prints the value of the first "=" token in +# . +field() +{ + local value + if ! value="$(awk -v key="$2" '{ + for (i = 1; i <= NF; i++) { + if (index($i, key "=") == 1) { + sub("^" key "=", "", $i) + print $i + found = 1 + exit + } + } + } + END { + if (!found) + exit 1 + }' <<< "$1")"; then + printf 'benchmark output missing %s field:\n%s\n' "$2" "$1" >&2 + return 1 + fi + printf '%s\n' "$value" +} + +numeric_median() +{ + printf '%s\n' "$@" | LC_ALL=C sort -n | awk '{ values[NR] = $1 } + END { + if (NR % 2) + print values[(NR + 1) / 2] + else + print (values[NR / 2] + values[NR / 2 + 1]) / 2 + }' +} + +run_a() +{ + printf 'A. one timed mmap / munmap fast-path pair per fresh elfuse process\n' + printf ' aggregate target: %s corrected CNTVCT ticks per call type\n' \ + "$ONESHOT_TARGET_TICKS" + printf '%-10s %8s %12s %12s\n' size guests 'mmap ns' 'munmap ns' + for size in "${SIZES[@]}"; do + mmap_ticks=0 + munmap_ticks=0 + runs=0 + ns_per_tick= + failed=0 + while awk -v mmap_ticks="$mmap_ticks" -v munmap_ticks="$munmap_ticks" \ + -v target="$ONESHOT_TARGET_TICKS" \ + 'BEGIN { exit ! (mmap_ticks < target || munmap_ticks < target) }'; do + if [ "$runs" -ge "$ONESHOT_MAX_RUNS" ]; then + printf 'one-shot benchmark exceeded %s runs for %s\n' \ + "$ONESHOT_MAX_RUNS" "$size" >&2 + return 1 + fi + if ! out="$("$ELFUSE" "$BENCH" a-one "$size" 2>&1)"; then + failed=1 + break + fi + raw_mmap_ticks="$(field "$out" mmap_ticks)" + raw_munmap_ticks="$(field "$out" munmap_ticks)" + read_ticks="$(field "$out" read_ticks)" + ns_per_tick="$(field "$out" ns_per_tick)" + mmap_ticks="$(awk -v total="$mmap_ticks" -v raw="$raw_mmap_ticks" \ + -v read="$read_ticks" 'BEGIN { print total + raw - read }')" + munmap_ticks="$(awk -v total="$munmap_ticks" \ + -v raw="$raw_munmap_ticks" -v read="$read_ticks" \ + 'BEGIN { print total + raw - read }')" + runs=$((runs + 1)) + done + if [ "$failed" -ne 0 ]; then + printf '%-10s %8s %12s %12s\n' "$(human "$size")" - FAILED - + continue + fi + mmap_ns="$(awk -v ticks="$mmap_ticks" -v runs="$runs" \ + -v ns="$ns_per_tick" 'BEGIN { print ticks * ns / runs }')" + munmap_ns="$(awk -v ticks="$munmap_ticks" -v runs="$runs" \ + -v ns="$ns_per_tick" 'BEGIN { print ticks * ns / runs }')" + printf '%-10s %8s %12.1f %12.1f\n' "$(human "$size")" "$runs" \ + "$mmap_ns" "$munmap_ns" + done + printf '\n' +} + +run_b() +{ + printf 'B. munmap after materializing one 4 KiB page (isolated)\n' + printf ' outer median: %s fresh elfuse processes per size\n' \ + "$MATERIALIZED_GUESTS" + printf '%-10s %8s %12s\n' size guests 'munmap ns' + for size in "${SIZES[@]}"; do + munmap_samples=() + failed=0 + for ((guest = 0; guest < MATERIALIZED_GUESTS; guest++)); do + if ! out="$("$ELFUSE" "$BENCH" b-one "$size" 2>&1)"; then + failed=1 + break + fi + munmap_samples+=("$(field "$out" munmap_ns)") + done + if [ "$failed" -ne 0 ]; then + printf '%-10s %8s %12s\n' "$(human "$size")" - FAILED + continue + fi + munmap_ns="$(numeric_median "${munmap_samples[@]}")" + printf '%-10s %8u %12.1f\n' "$(human "$size")" \ + "$MATERIALIZED_GUESTS" "$munmap_ns" + done + printf '\n' +} + +run_c() +{ + printf 'C. munmap after dirtying every 4 KiB page (isolated)\n' + printf '%-10s %8s %12s %12s %12s\n' size ops 'p50 ns' 'p95 ns' 'max ns' + for case in "${DIRTY_CASES[@]}"; do + size="${case%%:*}" + ops="${case##*:}" + if ! out="$("$ELFUSE" "$BENCH" c-one "$size" "$ops" 2>&1)"; then + printf '%-10s %8s %12s %12s %12s\n' "$(human "$size")" "$ops" \ + FAILED - - + continue + fi + p50_ns="$(field "$out" p50_ns)" + p95_ns="$(field "$out" p95_ns)" + max_ns="$(field "$out" max_ns)" + printf '%-10s %8s %12.1f %12.1f %12.1f\n' "$(human "$size")" "$ops" \ + "$p50_ns" "$p95_ns" "$max_ns" + done + printf '\n' +} + +run_d() +{ + "$ELFUSE" "$BENCH" host-drain "$HOST_DRAIN_SAMPLES" +} + +case "${1:-all}" in + a) run_a ;; + b) run_b ;; + c) run_c ;; + d) run_d ;; + all) run_a; run_b; run_c; run_d ;; + *) + echo "usage: $0 [a|b|c|d|all]" >&2 + exit 2 + ;; +esac diff --git a/tests/bench-mmap-lazy.c b/tests/bench-mmap-lazy.c new file mode 100644 index 00000000..6b4c2937 --- /dev/null +++ b/tests/bench-mmap-lazy.c @@ -0,0 +1,112 @@ +/* + * Guest microbenchmark for anonymous private mmap latency vs size. + * + * Measures mmap(), first-touch, and munmap() latency for MAP_PRIVATE | + * MAP_ANONYMOUS mappings from 4 KiB to 32 GiB. A lazy (deferred page-table) + * implementation should show size-independent mmap/munmap cost; an eager + * implementation scales linearly with length and exhausts resources on the + * multi-GiB sizes. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +static uint64_t now_ns(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t) ts.tv_sec * 1000000000ull + (uint64_t) ts.tv_nsec; +} + +static void bench_size(uint64_t size, int iters) +{ + uint64_t t_map = 0, t_touch = 0, t_unmap = 0; + int ok = 0; + + for (int i = 0; i < iters; i++) { + uint64_t t0 = now_ns(); + void *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t t1 = now_ns(); + if (p == MAP_FAILED) { + printf("%10llu KiB: mmap failed: %s\n", + (unsigned long long) (size >> 10), strerror(errno)); + return; + } + /* First touch: one write at the start and one mid-mapping. */ + volatile char *c = p; + c[0] = 1; + c[size / 2] = 1; + uint64_t t2 = now_ns(); + int rc = munmap(p, size); + uint64_t t3 = now_ns(); + if (rc != 0) { + printf("%10llu KiB: munmap failed: %s\n", + (unsigned long long) (size >> 10), strerror(errno)); + return; + } + t_map += t1 - t0; + t_touch += t2 - t1; + t_unmap += t3 - t2; + ok++; + } + printf( + "%10llu KiB: mmap %10llu ns touch2 %10llu ns munmap %10llu ns " + "(%d iters)\n", + (unsigned long long) (size >> 10), + (unsigned long long) (t_map / (uint64_t) ok), + (unsigned long long) (t_touch / (uint64_t) ok), + (unsigned long long) (t_unmap / (uint64_t) ok), ok); +} + +int main(int argc, char **argv) +{ + static const struct { + uint64_t size; + int iters; + } cases[] = { + {4ull << 10, 200}, {64ull << 10, 200}, {2ull << 20, 100}, + {64ull << 20, 20}, {512ull << 20, 10}, {2ull << 30, 5}, + {8ull << 30, 3}, {16ull << 30, 3}, {32ull << 30, 3}, + {64ull << 30, 1}, {128ull << 30, 1}, {256ull << 30, 1}, + }; + + /* Optional argv[1]: cap size in GiB (eager implementations commit host + * memory for every byte mapped; the full matrix would thrash small hosts). + */ + uint64_t cap = ~0ull; + if (argc > 1) + cap = (uint64_t) atoll(argv[1]) << 30; + + setvbuf(stdout, NULL, _IONBF, 0); + printf("anon private mmap latency vs size\n"); + for (unsigned i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) + if (cases[i].size <= cap) + bench_size(cases[i].size, cases[i].iters); + + /* Full-touch throughput sanity: 64 MiB written end to end. */ + uint64_t size = 64ull << 20; + uint64_t t0 = now_ns(); + void *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf("full-touch mmap failed: %s\n", strerror(errno)); + return 1; + } + memset(p, 0xa5, size); + uint64_t t1 = now_ns(); + munmap(p, size); + printf("mmap+memset 64MiB: %llu ns (%.2f GiB/s)\n", + (unsigned long long) (t1 - t0), + (double) size / 1.073741824 / (double) (t1 - t0)); + return 0; +} diff --git a/tests/bench-mmap-mt b/tests/bench-mmap-mt new file mode 100755 index 00000000..c738db4d --- /dev/null +++ b/tests/bench-mmap-mt @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# Measure concurrent mmap and munmap fast paths in fresh elfuse processes. + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ELFUSE="${ELFUSE:-${ROOT_DIR}/build/elfuse}" +BENCH="${BENCH_MMAP_BIN:-${ROOT_DIR}/build/bench-mmap}" +SAMPLES="${BENCH_MMAP_MT_SAMPLES:-7}" + +SIZES=(4096 2097152) +THREAD_COUNTS=(1 2 4 8) + +if [ ! -x "$ELFUSE" ] || [ ! -x "$BENCH" ]; then + echo "build build/elfuse and build/bench-mmap first" >&2 + exit 2 +fi +if ! [[ "$SAMPLES" =~ ^[1-9][0-9]*$ ]]; then + echo "BENCH_MMAP_MT_SAMPLES must be a positive integer" >&2 + exit 2 +fi + +human() +{ + awk -v n="$1" 'BEGIN { + if (n >= 1073741824) printf "%g GiB", n / 1073741824 + else if (n >= 1048576) printf "%g MiB", n / 1048576 + else printf "%g KiB", n / 1024 + }' +} + +field() +{ + local value + if ! value="$(awk -v key="$2" '{ + for (i = 1; i <= NF; i++) { + if (index($i, key "=") == 1) { + sub("^" key "=", "", $i) + print $i + found = 1 + exit + } + } + } + END { + if (!found) + exit 1 + }' <<< "$1")"; then + printf 'benchmark output missing %s field:\n%s\n' "$2" "$1" >&2 + return 1 + fi + printf '%s\n' "$value" +} + +numeric_median() +{ + printf '%s\n' "$@" | LC_ALL=C sort -n | awk '{ values[NR] = $1 } + END { + if (NR % 2) + print values[(NR + 1) / 2] + else + print (values[NR / 2] + values[NR / 2 + 1]) / 2 + }' +} + +printf 'Multi-threaded mmap/munmap fast paths\n' +printf 'Each row is the median of %s fresh elfuse processes.\n' "$SAMPLES" +printf 'ns is mean per-thread time; Mops/s is aggregate throughput.\n' +printf '%-10s %8s %8s %12s %12s %12s %12s\n' \ + size threads ops/thr 'mmap ns' 'mmap Mops/s' 'munmap ns' 'munmap Mops/s' + +for size in "${SIZES[@]}"; do + for threads in "${THREAD_COUNTS[@]}"; do + mmap_ns_samples=() + mmap_mops_samples=() + munmap_ns_samples=() + munmap_mops_samples=() + ops_per_thread=- + failed=0 + for ((sample = 0; sample < SAMPLES; sample++)); do + if ! out="$("$ELFUSE" "$BENCH" mt-one "$size" "$threads" 2>&1)"; then + printf 'benchmark failed for size=%s threads=%s:\n%s\n' \ + "$size" "$threads" "$out" >&2 + failed=1 + break + fi + if ! ops_per_thread="$(field "$out" ops_per_thread)" \ + || ! mmap_ns="$(field "$out" mmap_ns)" \ + || ! mmap_mops="$(field "$out" mmap_mops)" \ + || ! munmap_ns="$(field "$out" munmap_ns)" \ + || ! munmap_mops="$(field "$out" munmap_mops)"; then + failed=1 + break + fi + mmap_ns_samples+=("$mmap_ns") + mmap_mops_samples+=("$mmap_mops") + munmap_ns_samples+=("$munmap_ns") + munmap_mops_samples+=("$munmap_mops") + done + if [ "$failed" -ne 0 ]; then + printf '%-10s %8s %8s %12s\n' "$(human "$size")" "$threads" - FAILED + continue + fi + mmap_ns="$(numeric_median "${mmap_ns_samples[@]}")" + mmap_mops="$(numeric_median "${mmap_mops_samples[@]}")" + munmap_ns="$(numeric_median "${munmap_ns_samples[@]}")" + munmap_mops="$(numeric_median "${munmap_mops_samples[@]}")" + printf '%-10s %8s %8s %12.1f %12.3f %12.1f %12.3f\n' \ + "$(human "$size")" "$threads" "$ops_per_thread" "$mmap_ns" \ + "$mmap_mops" "$munmap_ns" "$munmap_mops" + done +done diff --git a/tests/bench-mmap.c b/tests/bench-mmap.c new file mode 100644 index 00000000..c037faa7 --- /dev/null +++ b/tests/bench-mmap.c @@ -0,0 +1,1079 @@ +/* + * Comprehensive anonymous-mmap microbenchmark for elfuse. + * + * Measures the guest-visible cost of the mmap subsystem in isolation: + * allocation, teardown, first-touch faults, permission splitting, and remap. It + * is self-contained -- no external harness -- and is meant to be run under + * elfuse (./build/elfuse ./build/bench-mmap) but also runs on any aarch64-linux + * host for a ground-truth comparison. + * + * Timing: reads CNTVCT_EL0 directly at EL0 (enabled by CNTKCTL_EL1.EL0VCTEN in + * bootstrap.c), so a measurement costs an isb + mrs, not a clock_gettime SVC. + * On Apple Silicon CNTFRQ is ~24 MHz (~41.7 ns/tick); amortizing over an + * adaptive batch drives the effective resolution well below one tick. This is + * the key fairness property: clock_gettime on a static guest falls through to + * the ~2 us SVC path and swamps any sub-us operation. + * + * Every in-process case takes one untimed warmup pass (to pay the one-time + * arena carve and page-table extension). Most sections report aggregate + * samples; section C retains every operation so its normal latency and long + * tail remain visible. Section A is driven host-side so every timed mmap and + * munmap pair gets a new elfuse process. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* CNTVCT timing */ + +static double ns_per_tick; + +static inline uint64_t rd(void) +{ + uint64_t v; + __asm__ volatile("isb\n\tmrs %0, cntvct_el0" : "=r"(v)); + return v; +} + +static void clock_init(void) +{ + uint64_t f; + __asm__ volatile("mrs %0, cntfrq_el0" : "=r"(f)); + if (f == 0) + f = 24000000; /* defensive: assume 24 MHz if RES0 */ + ns_per_tick = 1e9 / (double) f; +} + +static double ns(uint64_t ticks) +{ + return (double) ticks * ns_per_tick; +} + +static int cmp_d(const void *a, const void *b) +{ + double x = *(const double *) a, y = *(const double *) b; + return (x > y) - (x < y); +} + +static double median(double *v, int n) +{ + qsort(v, n, sizeof(*v), cmp_d); + return (n & 1) ? v[n / 2] : 0.5 * (v[n / 2 - 1] + v[n / 2]); +} + +/* R-7/sample quantile, matching the common (n - 1) * p interpolation. The input + * must already be sorted. + */ +static double sorted_quantile(const double *v, unsigned n, double p) +{ + double pos = (double) (n - 1) * p; + unsigned lo = (unsigned) pos; + unsigned hi = lo + (lo + 1 < n); + return v[lo] + (v[hi] - v[lo]) * (pos - lo); +} + +#define ITERS 15 +#define MIN_TIMED_TICKS 4096 +#define KIB (1ULL << 10) +#define MIB (1ULL << 20) +#define GIB (1ULL << 30) +#define DIRTY_VMEXIT_STRIDE (8 * MIB) + +/* Calibrate the rd()/rd() interval around a timed operation. Its median cost is + * subtracted from aggregate samples so the counter-read overhead is not + * attributed to mmap or munmap. + */ +static double rd_pair_ticks(void) +{ + enum { CAL_SAMPLES = 15, CAL_OPS = 4096 }; + double samples[CAL_SAMPLES]; + for (int sample = 0; sample < CAL_SAMPLES; sample++) { + uint64_t total = 0; + for (int op = 0; op < CAL_OPS; op++) { + uint64_t t0 = rd(); + uint64_t t1 = rd(); + total += t1 - t0; + } + samples[sample] = (double) total / CAL_OPS; + } + return median(samples, CAL_SAMPLES); +} + +static const char *human(uint64_t s, char *buf) +{ + if (s >= GIB) + sprintf(buf, "%llu GiB", (unsigned long long) (s / GIB)); + else if (s >= MIB) + sprintf(buf, "%llu MiB", (unsigned long long) (s / MIB)); + else + sprintf(buf, "%llu KiB", (unsigned long long) (s / KIB)); + return buf; +} + +/* A. One mmap and one munmap fast-path sample in a fresh guest. The host-side + * driver starts a new elfuse process for every invocation of this function. + * Prime the requested arena size outside the timed interval, then force a host + * drain so the arena is empty and its cursor is rewound. The two measured calls + * can then take the EL1 paths even when size exceeds the initial 64 MiB arena. + * Raw ticks are returned because one fast call is comparable to the counter + * period; the driver aggregates independent one-call samples before converting + * them to nanoseconds. + */ +static int bench_fastpath_once(uint64_t size) +{ + void *warmup = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (warmup == MAP_FAILED || munmap(warmup, size) != 0) + return 1; + + /* Drain the warmup retirement and rewind the now-empty arena. */ + (void) fcntl(-1, F_GETFD); + + uint64_t mmap_start = rd(); + void *ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t mmap_end = rd(); + if (ptr == MAP_FAILED) + return 1; + + uint64_t munmap_start = rd(); + int rc = munmap(ptr, size); + uint64_t munmap_end = rd(); + if (rc != 0) + return 1; + + printf( + "fast-once size=%llu mmap_ticks=%llu munmap_ticks=%llu " + "read_ticks=%.6f ns_per_tick=%.12f\n", + (unsigned long long) size, (unsigned long long) (mmap_end - mmap_start), + (unsigned long long) (munmap_end - munmap_start), rd_pair_ticks(), + ns_per_tick); + return 0; +} + +#define HOST_DRAIN_MAX_BATCH 30 +#define HOST_DRAIN_DEFAULT_SAMPLES 31 + +typedef struct { + double baseline_ns; + double mmap_extra_ns; + double mmap_ns_per_op; + double munmap_extra_ns; + double munmap_ns_per_op; +} host_drain_result_t; + +static bool timed_empty_vmexit(uint64_t *ticks) +{ + errno = 0; + uint64_t start = rd(); + int rc = fcntl(-1, F_GETFD); + uint64_t end = rd(); + if (rc != -1 || errno != EBADF) + return false; + *ticks = end - start; + return true; +} + +/* Pair a work-bearing fcntl VM exit with an empty one at the same mapping + * population. The difference leaves only the host's publication or retirement + * drain. Batches stop below both 32-entry ring limits so no mmap or munmap in + * the setup triggers an early HVC drain. + */ +static bool bench_host_drain_measure(uint64_t size, + unsigned batch, + unsigned samples, + host_drain_result_t *result) +{ + if (batch == 0 || batch > HOST_DRAIN_MAX_BATCH || samples == 0) + return false; + + void **mappings = calloc(batch, sizeof(*mappings)); + double *baseline = malloc(2 * samples * sizeof(*baseline)); + double *mmap_extra = malloc(samples * sizeof(*mmap_extra)); + double *munmap_extra = malloc(samples * sizeof(*munmap_extra)); + if (!mappings || !baseline || !mmap_extra || !munmap_extra) + goto fail; + + void *warmup = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t discard; + if (warmup == MAP_FAILED || munmap(warmup, size) != 0 || + !timed_empty_vmexit(&discard)) + goto fail; + + for (unsigned sample = 0; sample < samples; sample++) { + uint64_t mmap_base, mmap_work, munmap_base, munmap_work; + if (!timed_empty_vmexit(&mmap_base)) + goto fail; + + for (unsigned op = 0; op < batch; op++) { + mappings[op] = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mappings[op] == MAP_FAILED) + goto fail; + } + if (!timed_empty_vmexit(&mmap_work) || + !timed_empty_vmexit(&munmap_base)) + goto fail; + + for (unsigned op = 0; op < batch; op++) { + if (munmap(mappings[op], size) != 0) + goto fail; + mappings[op] = NULL; + } + if (!timed_empty_vmexit(&munmap_work)) + goto fail; + + baseline[2 * sample] = (double) mmap_base; + baseline[2 * sample + 1] = (double) munmap_base; + mmap_extra[sample] = (double) mmap_work - (double) mmap_base; + munmap_extra[sample] = (double) munmap_work - (double) munmap_base; + } + + result->baseline_ns = median(baseline, (int) (2 * samples)) * ns_per_tick; + result->mmap_extra_ns = median(mmap_extra, (int) samples) * ns_per_tick; + result->mmap_ns_per_op = result->mmap_extra_ns / batch; + result->munmap_extra_ns = median(munmap_extra, (int) samples) * ns_per_tick; + result->munmap_ns_per_op = result->munmap_extra_ns / batch; + free(munmap_extra); + free(mmap_extra); + free(baseline); + free(mappings); + return true; + +fail: + if (mappings) { + for (unsigned op = 0; op < batch; op++) { + if (mappings[op] && mappings[op] != MAP_FAILED) + munmap(mappings[op], size); + } + (void) fcntl(-1, F_GETFD); + } + free(munmap_extra); + free(mmap_extra); + free(baseline); + free(mappings); + return false; +} + +static bool bench_host_drain(unsigned samples) +{ + static const unsigned batches[] = {1, 2, 4, 8, 16, 30}; + const uint64_t size = 4 * KIB; + bool ok = true; + + printf("== D. host work drained at the next VM exit ==\n"); + printf("Empty fcntl VM exits provide the paired fixed-cost baseline.\n"); + printf("%-10s %8s %8s %12s %14s %12s %14s %12s\n", "size", "batch", + "samples", "baseline ns", "mmap extra ns", "mmap ns/op", + "munmap extra ns", "munmap ns/op"); + for (unsigned i = 0; i < sizeof(batches) / sizeof(batches[0]); i++) { + host_drain_result_t result; + char hb[16]; + if (!bench_host_drain_measure(size, batches[i], samples, &result)) { + printf("%-10s %8u %8u %14s\n", human(size, hb), batches[i], samples, + "FAILED"); + ok = false; + continue; + } + printf("%-10s %8u %8u %12.1f %14.1f %12.1f %14.1f %12.1f\n", + human(size, hb), batches[i], samples, result.baseline_ns, + result.mmap_extra_ns, result.mmap_ns_per_op, + result.munmap_extra_ns, result.munmap_ns_per_op); + } + printf("\n"); + return ok; +} + +/* Auxiliary: fresh bump-tail mmap isolates the lazy_fresh_range path. Allocate + * sequential run WITHOUT freeing, so every mapping lands at or above the arena + * high-water -- exactly the case lazy_fresh_range skips the stale-PTE scan for. + * Small mappings use the original 2-GiB footprint cap; large mappings use a + * minimum count chosen to retain multiple samples without exceeding 64 GiB of + * live fresh VA. Run this binary against an opt-off build to read the skip's + * contribution as the difference on this identical code path -- a MAP_FIXED + * "recycled" compare would instead measure the region-snapshot replacement + * path, not the skip. + */ +static void bench_fresh(void) +{ + static const uint64_t sizes[] = {4 * KIB, 64 * KIB, MIB, + 2 * MIB, 8 * MIB, 128 * MIB, + GIB, 8 * GIB, 32 * GIB}; + printf( + "== Auxiliary: fresh bump-tail mmap, per-mmap ns " + "(lazy_fresh_range path) ==\n"); + printf("%-10s %8s %14s\n", "size", "count", "fresh mmap ns"); + void *run[1000]; + for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); s++) { + uint64_t size = sizes[s]; + int n = (int) (2 * GIB / size); + if (n < 1) + n = 1; + if (n > 1000) + n = 1000; + if (size == GIB) + n = 8; + else if (size == 8 * GIB) + n = 4; + else if (size == 32 * GIB) + n = 2; + /* warmup one fresh mapping so the arena high-water is already primed */ + void *w = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (w != MAP_FAILED) + munmap(w, size); + uint64_t t0 = rd(); + for (int i = 0; i < n; i++) + run[i] = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t t1 = rd(); + int failed = 0; + for (int i = 0; i < n; i++) { + if (run[i] == MAP_FAILED) + failed++; + else + munmap(run[i], size); + } + char hb[16]; + if (failed) { + printf("%-10s %8d %14s (%d failed)\n", human(size, hb), n, + "PARTIAL", failed); + continue; + } + printf("%-10s %8d %14.1f\n", human(size, hb), n, ns(t1 - t0) / n); + } + printf("\n"); +} + +/* One fresh bump-tail run for the host-side driver. A new elfuse process is + * used for each invocation, so the driver can accumulate many counter ticks + * without exhausting one guest's VA space. + */ +static int bench_fresh_one(uint64_t size, int n) +{ + void *run[1000]; + if (n < 1 || n > (int) (sizeof(run) / sizeof(run[0]))) + return 2; + + void *warmup = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (warmup == MAP_FAILED) + return 1; + munmap(warmup, size); + + uint64_t t0 = rd(); + for (int i = 0; i < n; i++) + run[i] = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t t1 = rd(); + + int failed = 0; + for (int i = 0; i < n; i++) { + if (run[i] == MAP_FAILED) + failed++; + else + munmap(run[i], size); + } + if (failed) + return 1; + + printf( + "fresh size=%llu count=%d ticks=%llu read_ticks=%.6f " + "ns_per_tick=%.12f\n", + (unsigned long long) size, n, (unsigned long long) (t1 - t0), + rd_pair_ticks(), ns_per_tick); + return 0; +} + +/* Auxiliary: first-touch cost. Touch one byte per macOS 16 KiB host page. Each + * demands a distinct physical backing page, but it is not necessarily a + * distinct HVC: elfuse installs Stage-1 descriptors in 2 MiB windows and the + * fault-around policy may install several windows per exit. Report both the + * whole sweep and its per-host-page amortization; calling the latter a + * "per-fault" cost would substantially overcount guest translation faults. + */ +static void bench_fault(int pages, int drain_between) +{ + const uint64_t stride = 16 * KIB; + uint64_t size = stride * (uint64_t) (pages + 1); + printf( + "== Auxiliary: first-touch fault cost (16 KiB stride, %d pages%s) " + "==\n", + pages, drain_between ? ", forced retire drain" : ""); + double sweep[ITERS]; + for (int it = -1; it < ITERS; it++) { + volatile uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf(" mmap FAILED: %s\n\n", strerror(errno)); + return; + } + uint64_t t0 = rd(); + for (int i = 0; i < pages; i++) + p[(uint64_t) i * stride] = 1; + uint64_t t1 = rd(); + munmap((void *) p, size); + if (drain_between) + (void) fcntl(-1, F_GETFD); + if (it >= 0) + sweep[it] = ns(t1 - t0); + } + qsort(sweep, ITERS, sizeof(*sweep), cmp_d); + printf(" sweep: p50 %.1f us p95 %.1f us max %.1f us\n", + sorted_quantile(sweep, ITERS, 0.50) / 1000.0, + sorted_quantile(sweep, ITERS, 0.95) / 1000.0, + sweep[ITERS - 1] / 1000.0); + printf(" amortized/touch: p50 %.1f ns p95 %.1f ns max %.1f ns\n\n", + sorted_quantile(sweep, ITERS, 0.50) / pages, + sorted_quantile(sweep, ITERS, 0.95) / pages, + sweep[ITERS - 1] / pages); +} + +/* Auxiliary: mprotect split cost. Flip the middle 4 KiB of a 2 MiB RW block to + * PROT_READ, forcing guest_split_block to convert the L2 block into 512 L3 + * pages. Restore between iterations so each run does a fresh split. + */ +static void bench_mprotect_split(void) +{ + printf( + "== Auxiliary: mprotect split " + "(2 MiB block -> L3, protect middle 4 KiB) ==\n"); + double sp[ITERS]; + for (int it = -1; it < ITERS; it++) { + uint8_t *p = mmap(NULL, 2 * MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf(" mmap FAILED\n\n"); + return; + } + uint8_t *mid = p + MIB; + uint64_t t0 = rd(); + int rc = mprotect(mid, 4 * KIB, PROT_READ); + uint64_t t1 = rd(); + munmap(p, 2 * MIB); + if (rc != 0) { + printf(" mprotect FAILED: %s\n\n", strerror(errno)); + return; + } + if (it >= 0) + sp[it] = ns(t1 - t0); + } + printf(" split: median %.1f ns min %.1f ns\n\n", median(sp, ITERS), + sp[0]); +} + +/* Auxiliary: mremap grow, in-place vs forced move. */ +static void bench_mremap(void) +{ + printf("== Auxiliary: mremap grow 4 KiB -> 8 KiB ==\n"); + double inp[ITERS], mov[ITERS]; + + /* In-place: no blocker, the following page is free. */ + for (int it = -1; it < ITERS; it++) { + void *p = mmap(NULL, 4 * KIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf(" mmap FAILED\n\n"); + return; + } + uint64_t t0 = rd(); + void *q = mremap(p, 4 * KIB, 8 * KIB, MREMAP_MAYMOVE); + uint64_t t1 = rd(); + if (q == MAP_FAILED) { + munmap(p, 4 * KIB); + printf(" mremap in-place FAILED\n\n"); + return; + } + munmap(q, 8 * KIB); + if (it >= 0) + inp[it] = ns(t1 - t0); + } + + /* Forced move: a PROT_READ blocker sits immediately after, so the grow must + * relocate. + */ + for (int it = -1; it < ITERS; it++) { + uint8_t *p = mmap(NULL, 8 * KIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf(" mmap FAILED\n\n"); + return; + } + + /* free the tail page and pin it read-only so in-place growth is blocked + * but the head is still a 4 KiB mapping. + */ + munmap(p + 4 * KIB, 4 * KIB); + void *blk = mmap(p + 4 * KIB, 4 * KIB, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + uint64_t t0 = rd(); + void *q = mremap(p, 4 * KIB, 8 * KIB, MREMAP_MAYMOVE); + uint64_t t1 = rd(); + if (q == MAP_FAILED) { + printf(" mremap move FAILED\n\n"); + return; + } + munmap(q, 8 * KIB); + if (blk != MAP_FAILED) + munmap(blk, 4 * KIB); + if (it >= 0) + mov[it] = ns(t1 - t0); + } + printf(" in-place: median %.1f ns min %.1f ns\n", median(inp, ITERS), + inp[0]); + printf(" move: median %.1f ns min %.1f ns\n\n", median(mov, ITERS), + mov[0]); +} + +/* Auxiliary: multi-threaded mmap and munmap fast paths. Each worker primes its + * arena before the start barrier, allocates untouched mappings, then retires + * them after a second barrier. The host-side driver runs each configuration in + * a fresh elfuse process. + */ +typedef struct { + uint64_t size; + int n; + void **buf; + uint64_t mmap_start; + uint64_t mmap_end; + uint64_t munmap_start; + uint64_t munmap_end; + double mmap_ns; + double munmap_ns; + int failed; +} mt_arg_t; + +typedef struct { + double mmap_ns; + double mmap_mops; + double munmap_ns; + double munmap_mops; +} mt_result_t; + +static pthread_barrier_t mt_barrier; +static pthread_mutex_t mt_start_lock = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t mt_start_cond = PTHREAD_COND_INITIALIZER; +static int mt_start; +static int mt_abort; + +/* CNTVCT_EL0 reads a constant on worker vCPUs (EL0VCTEN is set for the main + * vCPU only), so the MT worker brackets its whole loop with clock_gettime and + * amortizes each SVC pair over n operations. + */ +static uint64_t mono_ns(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t) ts.tv_sec * 1000000000ull + (uint64_t) ts.tv_nsec; +} + +static void *mt_worker(void *p) +{ + mt_arg_t *a = p; + pthread_mutex_lock(&mt_start_lock); + while (!mt_start) + pthread_cond_wait(&mt_start_cond, &mt_start_lock); + int run = !mt_abort; + pthread_mutex_unlock(&mt_start_lock); + if (!run) + return NULL; + + void *warmup = mmap(NULL, a->size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (warmup == MAP_FAILED || munmap(warmup, a->size) != 0) + a->failed++; + (void) fcntl(-1, F_GETFD); + + pthread_barrier_wait(&mt_barrier); + a->mmap_start = mono_ns(); + for (int i = 0; i < a->n; i++) + a->buf[i] = mmap(NULL, a->size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + a->mmap_end = mono_ns(); + a->mmap_ns = (double) (a->mmap_end - a->mmap_start) / a->n; + + pthread_barrier_wait(&mt_barrier); + a->munmap_start = mono_ns(); + for (int i = 0; i < a->n; i++) { + if (a->buf[i] == MAP_FAILED) + a->failed++; + else if (munmap(a->buf[i], a->size) != 0) + a->failed++; + } + a->munmap_end = mono_ns(); + a->munmap_ns = (double) (a->munmap_end - a->munmap_start) / a->n; + return NULL; +} + +#define MT_MAX_THREADS 8 +#define MT_REGION_CAP 3000 /* keep T*n well under GUEST_MAX_REGIONS (4096) */ + +static int bench_mt_measure(uint64_t size, + int thread_count, + mt_result_t *result) +{ + int n = MT_REGION_CAP / thread_count; + mt_arg_t arg[MT_MAX_THREADS] = {0}; + pthread_t th[MT_MAX_THREADS]; + int ok = 1, created = 0; + for (int i = 0; i < thread_count; i++) { + arg[i].size = size; + arg[i].n = n; + arg[i].buf = calloc((size_t) n, sizeof(void *)); + if (!arg[i].buf) + ok = 0; + } + + int barrier_ready = 0; + if (ok) { + if (pthread_barrier_init(&mt_barrier, NULL, (unsigned) thread_count) == + 0) + barrier_ready = 1; + else + ok = 0; + } + pthread_mutex_lock(&mt_start_lock); + mt_start = 0; + mt_abort = 0; + pthread_mutex_unlock(&mt_start_lock); + for (int i = 0; i < thread_count && ok; i++) { + if (pthread_create(&th[i], NULL, mt_worker, &arg[i]) != 0) { + ok = 0; + break; + } + created++; + } + pthread_mutex_lock(&mt_start_lock); + mt_abort = !ok; + mt_start = 1; + pthread_cond_broadcast(&mt_start_cond); + pthread_mutex_unlock(&mt_start_lock); + + for (int i = 0; i < created; i++) + pthread_join(th[i], NULL); + + double mmap_sum = 0, munmap_sum = 0; + uint64_t mmap_start = UINT64_MAX, mmap_end = 0; + uint64_t munmap_start = UINT64_MAX, munmap_end = 0; + int failed = 0; + for (int i = 0; i < thread_count; i++) { + mmap_sum += arg[i].mmap_ns; + munmap_sum += arg[i].munmap_ns; + if (arg[i].mmap_start < mmap_start) + mmap_start = arg[i].mmap_start; + if (arg[i].mmap_end > mmap_end) + mmap_end = arg[i].mmap_end; + if (arg[i].munmap_start < munmap_start) + munmap_start = arg[i].munmap_start; + if (arg[i].munmap_end > munmap_end) + munmap_end = arg[i].munmap_end; + failed += arg[i].failed; + free(arg[i].buf); + } + if (barrier_ready) + pthread_barrier_destroy(&mt_barrier); + if (!ok || failed || mmap_end <= mmap_start || munmap_end <= munmap_start) + return 0; + + double total_ops = (double) n * thread_count; + result->mmap_ns = mmap_sum / thread_count; + result->mmap_mops = total_ops * 1000.0 / (double) (mmap_end - mmap_start); + result->munmap_ns = munmap_sum / thread_count; + result->munmap_mops = + total_ops * 1000.0 / (double) (munmap_end - munmap_start); + return 1; +} + +static void bench_mt(void) +{ + static const uint64_t sizes[] = {4 * KIB, 2 * MIB}; + static const int threads[] = {1, 2, 4, 8}; + printf("== Multi-threaded mmap/munmap fast paths ==\n"); + printf("ns is mean per-thread time; Mops/s is aggregate throughput.\n"); + printf("%-10s %8s %8s %12s %12s %12s %12s\n", "size", "threads", "ops/thr", + "mmap ns", "mmap Mops/s", "munmap ns", "munmap Mops/s"); + for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); s++) { + for (unsigned t = 0; t < sizeof(threads) / sizeof(threads[0]); t++) { + int thread_count = threads[t]; + int n = MT_REGION_CAP / thread_count; + mt_result_t result; + char hb[16]; + if (!bench_mt_measure(sizes[s], thread_count, &result)) { + printf("%-10s %8d %8d %12s\n", human(sizes[s], hb), + thread_count, n, "FAILED"); + continue; + } + printf("%-10s %8d %8d %12.1f %12.3f %12.1f %12.3f\n", + human(sizes[s], hb), thread_count, n, result.mmap_ns, + result.mmap_mops, result.munmap_ns, result.munmap_mops); + } + } + printf("\n"); +} + +/* B. Teardown after materialization. The store is deliberately outside the + * timed interval: it takes the lazy first-touch fault and installs the first + * page, then the counter brackets only munmap(). Each row has exactly one + * materialized 4-KiB page; touching every page would instead benchmark faulting + * and zeroing gigabytes of memory. + * + * bench_munmap_materialized_measure() holds the timing loop for exactly one + * size, shared by the in-process sweep below and the "b-one" isolated-process + * driver mode (see tests/bench-mmap-isolated). + */ +static int bench_munmap_materialized_measure(uint64_t size, + double timer_ticks, + unsigned *reported_ops, + double *munmap_ns) +{ + double unmap_ns[ITERS]; + int ok = 1; + for (int it = -1; it < ITERS && ok; it++) { + uint64_t unmap_ticks = 0; + unsigned ops = 0; + do { + volatile uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + ok = 0; + break; + } + p[0] = 1; /* materialize before starting the timed interval */ + uint64_t t0 = rd(); + int rc = munmap((void *) p, size); + uint64_t t1 = rd(); + unmap_ticks += t1 - t0; + if (rc != 0) { + ok = 0; + break; + } + ops++; + } while (unmap_ticks < MIN_TIMED_TICKS); + + if (it >= 0 && ok) { + unmap_ns[it] = + ((double) unmap_ticks / ops - timer_ticks) * ns_per_tick; + *reported_ops = ops; + } + } + if (!ok) + return 0; + *munmap_ns = median(unmap_ns, ITERS); + return 1; +} + +static void bench_munmap_materialized(void) +{ + double timer_ticks = rd_pair_ticks(); + static const uint64_t sizes[] = { + 4 * KIB, 16 * KIB, 64 * KIB, 256 * KIB, MIB, 2 * MIB, 8 * MIB, + 64 * MIB, 256 * MIB, GIB, 4 * GIB, 16 * GIB, 32 * GIB, + }; + + printf("== B. munmap after materializing one 4 KiB page ==\n"); + printf("%-10s %8s %12s\n", "size", "ops", "munmap ns"); + for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); s++) { + uint64_t size = sizes[s]; + unsigned reported_ops = 0; + double munmap_ns = 0; + char hb[16]; + if (!bench_munmap_materialized_measure(size, timer_ticks, &reported_ops, + &munmap_ns)) + printf("%-10s %8s %12s\n", human(size, hb), "-", "FAILED"); + else + printf("%-10s %8u %12.1f\n", human(size, hb), reported_ops, + munmap_ns); + } + printf("\n"); +} + +/* C. Teardown after every page was dirtied. Page stores happen before the timed + * interval, so this reports only munmap's handling of the materialized, dirty + * mapping. Each size has a fixed operation count and every munmap is retained + * separately. In particular, one slow first operation cannot end an adaptive + * batch and become the whole sample. Counts decrease with size to bound total + * dirtying work. The 1-GiB case uses eight operations, and the untimed store + * loop takes periodic VM exits so every vCPU-run interval stays below elfuse's + * watchdog; interpolated p95 remains distinct from max. + * + * bench_munmap_dirty_measure() holds the timing loop for exactly one size, + * shared by the in-process sweep below and the "c-one" isolated-process driver + * mode (see tests/bench-mmap-isolated). + */ +static int bench_munmap_dirty_measure(uint64_t size, + unsigned ops, + double timer_ticks, + double *p50_ns, + double *p95_ns, + double *max_ns) +{ + double *unmap_ns = malloc((size_t) ops * sizeof(*unmap_ns)); + int ok = 1; + if (!unmap_ns) + ok = 0; + + /* One full untimed warmup pays setup without consuming an observation. */ + for (int64_t op = -1; op < (int64_t) ops && ok; op++) { + volatile uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + ok = 0; + break; + } + for (uint64_t off = 0; off < size; off += 4 * KIB) { + p[off] = (uint8_t) (off >> 12); + + /* Large already-backed runs can execute in EL0 long enough for the + * benchmark's 10-second vCPU watchdog to fire under macOS memory + * pressure. This deliberately failing fcntl is a guaranteed, + * side-effect-free HVC outside the timed region. It also drains the + * preceding retirement in bounded chunks. + */ + if (off != 0 && (off & (DIRTY_VMEXIT_STRIDE - 1)) == 0) + (void) fcntl(-1, F_GETFD); + } + + /* Establish an identical clean host-retirement boundary before every + * measurement. The interval below then contains EL1's descriptor/TLBI + * path, never the previous operation's cleanup. + */ + (void) fcntl(-1, F_GETFD); + + uint64_t t0 = rd(); + int rc = munmap((void *) p, size); + uint64_t t1 = rd(); + if (rc != 0) { + ok = 0; + break; + } + if (op >= 0) { + double ticks = (double) (t1 - t0) - timer_ticks; + if (ticks < 0.0) + ticks = 0.0; + unmap_ns[op] = ticks * ns_per_tick; + } + } + + if (!ok) { + free(unmap_ns); + return 0; + } + qsort(unmap_ns, ops, sizeof(*unmap_ns), cmp_d); + *p50_ns = sorted_quantile(unmap_ns, ops, 0.50); + *p95_ns = sorted_quantile(unmap_ns, ops, 0.95); + *max_ns = unmap_ns[ops - 1]; + free(unmap_ns); + return 1; +} + +static void bench_munmap_dirty(void) +{ + static const struct { + uint64_t size; + unsigned ops; + } cases[] = { + {4 * KIB, 2048}, {16 * KIB, 2048}, {64 * KIB, 2048}, {256 * KIB, 1024}, + {MIB, 512}, {2 * MIB, 256}, {8 * MIB, 128}, {64 * MIB, 32}, + {256 * MIB, 24}, {GIB, 8}, + }; + double timer_ticks = rd_pair_ticks(); + + printf("== C. munmap after dirtying every 4 KiB page ==\n"); + printf("%-10s %8s %12s %12s %12s\n", "size", "ops", "p50 ns", "p95 ns", + "max ns"); + for (unsigned s = 0; s < sizeof(cases) / sizeof(cases[0]); s++) { + uint64_t size = cases[s].size; + unsigned ops = cases[s].ops; + double p50_ns = 0, p95_ns = 0, max_ns = 0; + char hb[16]; + if (!bench_munmap_dirty_measure(size, ops, timer_ticks, &p50_ns, + &p95_ns, &max_ns)) + printf("%-10s %8u %12s %12s %12s\n", human(size, hb), ops, "FAILED", + "-", "-"); + else + printf("%-10s %8u %12.1f %12.1f %12.1f\n", human(size, hb), ops, + p50_ns, p95_ns, max_ns); + } + printf("\n"); +} + +int main(int argc, char **argv) +{ + clock_init(); + if ((argc == 2 || argc == 3) && strcmp(argv[1], "host-drain") == 0) { + unsigned samples = HOST_DRAIN_DEFAULT_SAMPLES; + if (argc == 3) { + char *end = NULL; + errno = 0; + unsigned long value = strtoul(argv[2], &end, 0); + if (errno || !end || *end || value == 0 || value > 100000) + return 2; + samples = (unsigned) value; + } + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + return bench_host_drain(samples) ? 0 : 1; + } + if (argc == 2 && strcmp(argv[1], "b") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_munmap_materialized(); + return 0; + } + if (argc == 2 && strcmp(argv[1], "c") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_munmap_dirty(); + return 0; + } + if (argc == 2 && strcmp(argv[1], "mt") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_mt(); + return 0; + } + if (argc == 4 && strcmp(argv[1], "mt-one") == 0) { + char *end = NULL; + errno = 0; + uint64_t size = strtoull(argv[2], &end, 0); + if (errno || !end || *end || size == 0) + return 2; + errno = 0; + long threads_arg = strtol(argv[3], &end, 0); + if (errno || !end || *end || + (threads_arg != 1 && threads_arg != 2 && threads_arg != 4 && + threads_arg != 8)) + return 2; + int thread_count = (int) threads_arg; + mt_result_t result; + if (!bench_mt_measure(size, thread_count, &result)) { + printf("fastpath-mt size=%llu threads=%d FAILED\n", + (unsigned long long) size, thread_count); + return 1; + } + printf( + "fastpath-mt size=%llu threads=%d ops_per_thread=%d " + "mmap_ns=%.6f mmap_mops=%.6f munmap_ns=%.6f " + "munmap_mops=%.6f\n", + (unsigned long long) size, thread_count, + MT_REGION_CAP / thread_count, result.mmap_ns, result.mmap_mops, + result.munmap_ns, result.munmap_mops); + return 0; + } + if (argc == 2 && strcmp(argv[1], "fault") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_fault(512, 0); + return 0; + } + if (argc == 3 && strcmp(argv[1], "fault") == 0) { + char *end = NULL; + errno = 0; + long pages = strtol(argv[2], &end, 0); + if (errno || !end || *end || pages < 1 || pages > 1048576) + return 2; + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_fault((int) pages, 0); + return 0; + } + if (argc == 2 && strcmp(argv[1], "fault-drain") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_fault(512, 1); + return 0; + } + if (argc == 4 && strcmp(argv[1], "fresh") == 0) { + char *end = NULL; + errno = 0; + uint64_t size = strtoull(argv[2], &end, 0); + if (errno || !end || *end || size == 0) + return 2; + errno = 0; + long count = strtol(argv[3], &end, 0); + if (errno || !end || *end || count < 1 || count > 1000) + return 2; + return bench_fresh_one(size, (int) count); + } + + /* One section-A sample. The host-side driver starts a new elfuse process + * for every invocation and aggregates the raw one-call timings. + */ + if (argc == 3 && strcmp(argv[1], "a-one") == 0) { + char *end = NULL; + errno = 0; + uint64_t size = strtoull(argv[2], &end, 0); + if (errno || !end || *end || size == 0) + return 2; + return bench_fastpath_once(size); + } + /* One section-B data point for the host-side driver. */ + if (argc == 3 && strcmp(argv[1], "b-one") == 0) { + char *end = NULL; + errno = 0; + uint64_t size = strtoull(argv[2], &end, 0); + if (errno || !end || *end || size == 0) + return 2; + double timer_ticks = rd_pair_ticks(); + unsigned reported_ops = 0; + double munmap_ns = 0; + if (!bench_munmap_materialized_measure(size, timer_ticks, &reported_ops, + &munmap_ns)) { + printf("munmap-materialized size=%llu FAILED\n", + (unsigned long long) size); + return 1; + } + printf("munmap-materialized size=%llu ops=%u munmap_ns=%.6f\n", + (unsigned long long) size, reported_ops, munmap_ns); + return 0; + } + + /* One section-C data point for the host-side driver. The op count is + * explicit because C's cases[] table pairs it to size, and the driver must + * pass the same value the table would give. + */ + if (argc == 4 && strcmp(argv[1], "c-one") == 0) { + char *end = NULL; + errno = 0; + uint64_t size = strtoull(argv[2], &end, 0); + if (errno || !end || *end || size == 0) + return 2; + errno = 0; + long ops_arg = strtol(argv[3], &end, 0); + if (errno || !end || *end || ops_arg < 1 || ops_arg > 1000000) + return 2; + unsigned ops = (unsigned) ops_arg; + double timer_ticks = rd_pair_ticks(); + double p50_ns = 0, p95_ns = 0, max_ns = 0; + if (!bench_munmap_dirty_measure(size, ops, timer_ticks, &p50_ns, + &p95_ns, &max_ns)) { + printf("munmap-dirty size=%llu FAILED\n", + (unsigned long long) size); + return 1; + } + printf( + "munmap-dirty size=%llu ops=%u p50_ns=%.6f p95_ns=%.6f " + "max_ns=%.6f\n", + (unsigned long long) size, ops, p50_ns, p95_ns, max_ns); + return 0; + } + if (argc != 1) + return 2; + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_munmap_materialized(); + bench_munmap_dirty(); + bench_fresh(); + bench_fault(512, 0); + bench_mprotect_split(); + bench_mremap(); + bench_mt(); + return 0; +} diff --git a/tests/manifest.txt b/tests/manifest.txt index 899c0564..8cd62859 100644 --- a/tests/manifest.txt +++ b/tests/manifest.txt @@ -123,6 +123,10 @@ test-guard-page test-mmap-hint test-mmap-sigbus-efault +[section] Lazy anonymous mmap tests +test-mmap-lazy +test-mmap-fastpath + [section] mremap tests test-mremap test-mremap-infra diff --git a/tests/test-fork-ipc-protocol-host.c b/tests/test-fork-ipc-protocol-host.c index 30d64bde..f36be99f 100644 --- a/tests/test-fork-ipc-protocol-host.c +++ b/tests/test-fork-ipc-protocol-host.c @@ -22,9 +22,10 @@ #define PREVIOUS_ELFN_MAGIC 0x454C464EU #define PREVIOUS_ELFO_MAGIC 0x454C464FU #define PREVIOUS_ELFP_MAGIC 0x454C4650U +#define PREVIOUS_ELFQ_MAGIC 0x454C4651U -_Static_assert(FORK_IPC_PROTOCOL_MAGIC == 0x454C4651U, - "fork IPC protocol magic must remain ELFQ until the next " +_Static_assert(FORK_IPC_PROTOCOL_MAGIC == 0x454C4652U, + "fork IPC protocol magic must remain ELFR until the next " "incompatible wire-format change"); _Static_assert(IPC_MAGIC_HEADER == FORK_IPC_PROTOCOL_MAGIC, "header magic must be the protocol identity"); @@ -40,6 +41,8 @@ _Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFO_MAGIC, "per-fd description ownership requires rejecting ELFO peers"); _Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFP_MAGIC, "the per-fd intercept marker requires rejecting ELFP peers"); +_Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFQ_MAGIC, + "the dirty bitmap wire requires rejecting ELFQ peers"); _Static_assert(IPC_MAGIC_SENTINEL != FORK_IPC_PROTOCOL_MAGIC, "process-state sentinel must not alias the header protocol"); diff --git a/tests/test-matrix.sh b/tests/test-matrix.sh index fe6a5111..8e630bcf 100755 --- a/tests/test-matrix.sh +++ b/tests/test-matrix.sh @@ -643,22 +643,22 @@ test_pipe() # # This is the full aarch64 unit-test surface: every tests/manifest.txt ("make # check") binary except the handful that assert elfuse-internal implementation -# details with no meaningful counterpart on a real kernel (the EL1 shim -# fast-path suite -- test-shim-* and test-shim-cred-race, which probe elfuse's -# own shim_data block and identity cache -- test-mremap-infra, which guards -# elfuse's guest-IPA infra reserve, and test-oom-proc, documented in its own -# header). test-mremap-tail-emfile is listed here as an elfuse-lane regression -# and marked QEMU_SKIP because its host-reserve assertion has no Linux analogue. -# There is no "core" vs "extended" split here; everything below runs in both -# elfuse-aarch64 and qemu-aarch64 modes, and genuine, understood divergences -# from the qemu reference kernel are called out via QEMU_SKIP with a comment -# rather than silently dropped from this list. The unit lane runs binaries this -# repo builds, not fixtures it downloads, so an empty build/ is a setup mistake -# rather than a run to report on. Without this every test fails on a missing -# file and the summary reads like a hundred-odd regressions; "make clean" -# followed by "make elfuse" is enough to produce it, because elfuse alone does -# not build the test binaries. Name the cause once and stop, the way driver.sh -# does with ALLOW_MISSING_BINARIES. +# details with no meaningful counterpart on a real kernel. Test-shim-* and +# test-shim-cred-race probe elfuse's own shim_data block and identity cache; +# test-mmap-fastpath probes elfuse's arena and TLBI protocol; test-mremap-infra +# guards elfuse's guest-IPA infra reserve; test-oom-proc documents its reason in +# its own header. test-mremap-tail-emfile is listed here as an elfuse-lane +# regression and marked QEMU_SKIP because its host-reserve assertion has no +# Linux analogue. There is no "core" vs "extended" split here; everything below +# runs in both elfuse-aarch64 and qemu-aarch64 modes, and genuine, understood +# divergences from the qemu reference kernel are called out via QEMU_SKIP with a +# comment rather than silently dropped from this list. The unit lane runs +# binaries this repo builds, not fixtures it downloads, so an empty build/ is a +# setup mistake rather than a run to report on. Without this every test fails on +# a missing file and the summary reads like a hundred-odd regressions; "make +# clean" followed by "make elfuse" is enough to produce it, because elfuse alone +# does not build the test binaries. Name the cause once and stop, the way +# driver.sh does with ALLOW_MISSING_BINARIES. require_unit_binaries() { local bindir="$1" @@ -843,6 +843,7 @@ run_unit_tests() test_rc "$runner" "test-mmap-hint" 0 "$bindir/test-mmap-hint" test_rc "$runner" "test-mmap-sigbus-efault" 0 "$bindir/test-mmap-sigbus-efault" + test_rc "$runner" "test-mmap-lazy" 0 "$bindir/test-mmap-lazy" printf "\nLow-base ET_EXEC memory regression\n" test_rc "$runner" "test-lowbase-mem-200000" 0 "$bindir/test-lowbase-mem-200000" diff --git a/tests/test-mmap-dirty-stats.sh b/tests/test-mmap-dirty-stats.sh new file mode 100755 index 00000000..9d8e2715 --- /dev/null +++ b/tests/test-mmap-dirty-stats.sh @@ -0,0 +1,45 @@ +#!/bin/sh +# Counter-backed dirty-map materialization integration checks. + +set -eu + +ELFUSE=${1:-build/elfuse} +TEST_BIN=${2:-build/test-mmap-lazy} +TMPDIR_CASE=$(mktemp -d "${TMPDIR:-/tmp}/elfuse-dirty-map.XXXXXX") +trap 'rm -rf "$TMPDIR_CASE"' EXIT INT TERM + +ELFUSE_SHIM_STATS=1 "$ELFUSE" "$TEST_BIN" \ + > "$TMPDIR_CASE/out" 2> "$TMPDIR_CASE/err" + +counter() +{ + key=$1 + awk -v key="$key" \ + '$1 == key { print $2; found = 1 } END { if (!found) exit 1 }' \ + "$TMPDIR_CASE/err" +} + +require_ge() +{ + key=$1 + floor=$2 + value=$(counter "$key") || { + printf 'missing dirty-map counter %s\n' "$key" >&2 + return 1 + } + if [ "$value" -lt "$floor" ]; then + printf '%s=%s, expected >= %s\n' "$key" "$value" "$floor" >&2 + return 1 + fi +} + +require_ge FAULT_CLEAN_SKIP 1 +require_ge FAULT_DIRTY_MEMSET 1 +require_ge FAULT_ALREADY_VALID 1 +require_ge FAULT_WINDOW_BYTES 2097152 + +printf ' clean-block zero skip OK\n' +printf ' dirty-block selective memset OK\n' +printf ' already-valid early return OK\n' +printf ' materialized-window bytes OK\n' +printf 'test-mmap-dirty-stats: PASS\n' diff --git a/tests/test-mmap-fastpath-stats.sh b/tests/test-mmap-fastpath-stats.sh new file mode 100755 index 00000000..ef50f15c --- /dev/null +++ b/tests/test-mmap-fastpath-stats.sh @@ -0,0 +1,162 @@ +#!/bin/sh +# Counter-backed refill, adaptive sizing, giant-request guard, and VA recycle +# integration checks for the EL1 anonymous-mmap consumer fast path. + +set -eu + +ELFUSE=${1:-build/elfuse} +TEST_BIN=${2:-build/test-mmap-fastpath} +TMPDIR_CASE=$(mktemp -d "${TMPDIR:-/tmp}/elfuse-mmap-stats.XXXXXX") +trap 'rm -rf "$TMPDIR_CASE"' EXIT INT TERM + +run_case() +{ + case_name=$1 + out="$TMPDIR_CASE/$case_name.out" + err="$TMPDIR_CASE/$case_name.err" + ELFUSE_SHIM_STATS=1 "$ELFUSE" "$TEST_BIN" "--stats-$case_name" \ + > "$out" 2> "$err" +} + +counter() +{ + case_name=$1 + key=$2 + value=$(awk -v key="$key" '$1 == key { print $2; found = 1 } END { if (!found) exit 1 }' \ + "$TMPDIR_CASE/$case_name.err") || { + printf 'missing counter %s in case %s\n' "$key" "$case_name" >&2 + return 1 + } + printf '%s\n' "$value" +} + +require_ge() +{ + case_name=$1 + key=$2 + floor=$3 + value=$(counter "$case_name" "$key") + if [ "$value" -lt "$floor" ]; then + printf '%s: %s=%s, expected >= %s\n' \ + "$case_name" "$key" "$value" "$floor" >&2 + return 1 + fi +} + +require_eq() +{ + case_name=$1 + key=$2 + expected=$3 + value=$(counter "$case_name" "$key") + if [ "$value" -ne "$expected" ]; then + printf '%s: %s=%s, expected %s\n' \ + "$case_name" "$key" "$value" "$expected" >&2 + return 1 + fi +} + +require_le() +{ + case_name=$1 + key=$2 + ceiling=$3 + value=$(counter "$case_name" "$key") + if [ "$value" -gt "$ceiling" ]; then + printf '%s: %s=%s, expected <= %s\n' \ + "$case_name" "$key" "$value" "$ceiling" >&2 + return 1 + fi +} + +run_case ring-full +require_ge ring-full MMAP_HIT 32 +require_ge ring-full MMAP_RING_FULL 1 +printf ' 32-entry ring fallback OK\n' + +run_case np2-10m +require_ge np2-10m MMAP_HIT 80 +require_ge np2-10m MMAP_CAPACITY_MISS 1 +printf ' sustained 10 MiB stream OK\n' + +run_case np2-48m +require_ge np2-48m MMAP_HIT 40 +require_ge np2-48m MMAP_CAPACITY_MISS 1 +printf ' sustained 48 MiB stream OK\n' + +run_case np2-100m +require_ge np2-100m MMAP_HIT 24 +require_ge np2-100m MMAP_CAPACITY_MISS 1 +printf ' sustained 100 MiB stream OK\n' + +run_case escalation +require_ge escalation MMAP_HIT 45 +require_eq escalation MMAP_ARENA_CURRENT 17179869184 +printf ' 10 MiB -> 512 MiB escalation OK\n' + +run_case giant-guard +require_ge giant-guard MMAP_HIT 34 +require_eq giant-guard MMAP_ARENA_PEAK 34359738368 +printf ' 2 GiB request uses fast path OK\n' + +run_case adaptive-small +require_eq adaptive-small MMAP_ARENA_CURRENT 67108864 +require_eq adaptive-small MMAP_ARENA_PEAK 67108864 +printf ' small-stream arena floor OK\n' + +run_case adaptive-retention +require_eq adaptive-retention MMAP_ARENA_CURRENT 17179869184 +require_eq adaptive-retention MMAP_ARENA_PEAK 17179869184 +printf ' large arena retained OK\n' + +run_case adaptive-rewind-growth +require_ge adaptive-rewind-growth MMAP_CAPACITY_MISS 1 +require_eq adaptive-rewind-growth MMAP_ARENA_CURRENT 268435456 +printf ' rewound arena grows to target OK\n' + +run_case recycle +require_ge recycle MMAP_RECYCLE 1 +require_le recycle MMAP_HIGH_WATER 201326592 +printf ' arena VA recycling OK\n' + +# Mixed-size churn now recycles VA through the arena rewind and the host gap +# allocator alone. MMAP_RECYCLE is the load-bearing assertion: it is nonzero +# only when a refill actually reclaimed a hole below the high-water mark, so it +# fails the moment VA recovery stops and the allocator only walks forward. The +# hit count keeps the case honest -- the other two would also pass if the fast +# path stopped being taken at all -- and the bound catches growth that recovery +# is too slow to contain. +run_case mixed-churn +require_ge mixed-churn MMAP_HIT 100 +require_ge mixed-churn MMAP_RECYCLE 1 +require_le mixed-churn MMAP_HIGH_WATER 402653184 +printf ' mixed-size churn recycles VA OK\n' + +# A first touch enters the host while the 32 GiB mapping is still live. Its +# arena descriptor must survive that VM exit so the following munmap remains an +# EL1 retirement instead of stranding the mapping in a speculative top-up. +run_case large-materialized +require_eq large-materialized MMAP_ARENA_PEAK 34359738368 +require_le large-materialized MMAP_HIGH_WATER 38654705664 +printf ' large live arena is retained OK\n' + +# Fork drains and revokes every arena. A near-empty current arena must not be +# topped up between those operations; only the initial vCPU arena is counted. +run_case fork-no-topup +require_eq fork-no-topup MMAP_HIT 1 +require_eq fork-no-topup MMAP_REFILL 1 +printf ' fork skips arena top-up OK\n' + +run_case invalid-futex +require_eq invalid-futex FAULT_WINDOW_BYTES 0 +printf ' invalid futex stays lazy OK\n' + +# The test synchronizes through atomics while sibling vCPUs retain control, so +# msync, W^X, and maps/smaps are the first host observers of fast publications +# and retirements. Keep the case counter-backed so a disabled fast path cannot +# turn the semantic checks into a false green. +run_case observer-semantics +require_ge observer-semantics MMAP_HIT 1 +printf ' observer semantics under churn OK\n' + +printf 'test-mmap-fastpath-stats: PASS\n' diff --git a/tests/test-mmap-fastpath.c b/tests/test-mmap-fastpath.c new file mode 100644 index 00000000..74f38263 --- /dev/null +++ b/tests/test-mmap-fastpath.c @@ -0,0 +1,982 @@ +/* + * EL1 consumer-mmap fast-path integration tests. + * + * Run through the dedicated make target without an ELFUSE_MMAP_FASTPATH + * override so the default-enabled configuration is exercised. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" +#include "raw-syscall.h" + +int passes = 0, fails = 0; + +static sigjmp_buf segv_jmp; + +static inline void spin_hint(void); + +static void segv_handler(int sig) +{ + (void) sig; + siglongjmp(segv_jmp, 1); +} + +static int proc_extent_for(const char *path, + uintptr_t needle, + uintptr_t *lo_out, + uintptr_t *hi_out) +{ + int fd = open(path, O_RDONLY); + if (fd < 0) + return -1; + + size_t cap = 16384, used = 0; + char *buf = malloc(cap); + if (!buf) { + close(fd); + return -1; + } + for (;;) { + if (used == cap - 1) { + if (cap >= 4 * 1024 * 1024) { + free(buf); + close(fd); + return -1; + } + size_t next_cap = cap * 2; + char *next = realloc(buf, next_cap); + if (!next) { + free(buf); + close(fd); + return -1; + } + buf = next; + cap = next_cap; + } + ssize_t n = read(fd, buf + used, cap - used - 1); + if (n < 0 && errno == EINTR) + continue; + if (n < 0) { + free(buf); + close(fd); + return -1; + } + if (n == 0) + break; + used += (size_t) n; + } + close(fd); + if (used == 0) { + free(buf); + return -1; + } + buf[used] = '\0'; + + char *line = buf; + while (*line) { + unsigned long long lo, hi; + if (sscanf(line, "%llx-%llx", &lo, &hi) == 2 && needle >= lo && + needle < hi) { + *lo_out = (uintptr_t) lo; + *hi_out = (uintptr_t) hi; + free(buf); + return 1; + } + char *nl = strchr(line, '\n'); + if (!nl) + break; + line = nl + 1; + } + free(buf); + return 0; +} + +static int maps_extent_for(uintptr_t needle, + uintptr_t *lo_out, + uintptr_t *hi_out) +{ + return proc_extent_for("/proc/self/maps", needle, lo_out, hi_out) == 1 ? 0 + : -1; +} + +static void test_fidelity(void) +{ + TEST("unconsumed arena is absent and faults"); + struct sigaction sa = {.sa_handler = segv_handler}; + struct sigaction old_sa; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGSEGV, &sa, &old_sa) != 0) { + FAIL("sigaction"); + return; + } + + uint8_t *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + p[0] = 0x5a; /* drains the publication through the fault-side lock */ + + volatile uint8_t *unconsumed = p + 4096; + if (sigsetjmp(segv_jmp, 1) == 0) { + (void) *unconsumed; + FAIL("wild read into unconsumed arena did not SIGSEGV"); + munmap(p, 4096); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + + uintptr_t lo = 0, hi = 0; + if (maps_extent_for((uintptr_t) p, &lo, &hi) < 0 || lo != (uintptr_t) p || + hi != (uintptr_t) p + 4096) { + FAIL("/proc/self/maps exposed more than the consumed page"); + munmap(p, 4096); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + if (munmap(p, 4096) != 0) { + FAIL("munmap"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + + /* No syscall may intervene between munmap and this load: EL1 must have + * invalidated the Stage-1 descriptor and completed broadcast TLBI before + * returning, even though host region cleanup is still deferred. + */ + if (sigsetjmp(segv_jmp, 1) == 0) { + (void) *(volatile uint8_t *) p; + FAIL("access immediately after munmap did not SIGSEGV"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + sigaction(SIGSEGV, &old_sa, NULL); + PASS(); +} + +static void test_exhaustion_fallback(void) +{ + TEST("arena exhaustion falls back to host mmap"); + const size_t len = 80ULL << 20; /* larger than the first 64MiB arena */ + uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("80MiB mmap"); + return; + } + if (p[0] != 0 || p[len - 1] != 0) { + FAIL("fallback mapping was not zero-filled"); + munmap(p, len); + return; + } + p[0] = 1; + p[len - 1] = 2; + if (munmap(p, len) != 0) { + FAIL("munmap"); + return; + } + PASS(); +} + +typedef struct { + volatile uint8_t *p; + _Atomic int ready; + _Atomic int go; + _Atomic int result; +} sibling_tlbi_arg_t; + +static void *sibling_tlbi_worker(void *opaque) +{ + sibling_tlbi_arg_t *arg = opaque; + if (sigsetjmp(segv_jmp, 1) == 0) { + (void) arg->p[0]; /* seed a translation on this sibling vCPU */ + atomic_store_explicit(&arg->ready, 1, memory_order_release); + while (!atomic_load_explicit(&arg->go, memory_order_acquire)) + spin_hint(); + (void) arg->p[0]; + atomic_store_explicit(&arg->result, -1, memory_order_release); + } else { + atomic_store_explicit(&arg->result, 1, memory_order_release); + } + return NULL; +} + +static const char *run_sibling_tlbi(size_t len, size_t materialize_stride) +{ + struct sigaction sa = {.sa_handler = segv_handler}; + struct sigaction old_sa; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGSEGV, &sa, &old_sa) != 0) + return "sigaction"; + + volatile uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + sigaction(SIGSEGV, &old_sa, NULL); + return "mmap"; + } + + for (size_t off = 0; off < len; off += materialize_stride) + p[off] = (uint8_t) (off >> 21); + + sibling_tlbi_arg_t arg = {.p = p}; + pthread_t worker; + if (pthread_create(&worker, NULL, sibling_tlbi_worker, &arg) != 0) { + munmap((void *) p, len); + sigaction(SIGSEGV, &old_sa, NULL); + return "pthread_create"; + } + while (!atomic_load_explicit(&arg.ready, memory_order_acquire)) + spin_hint(); + + if (munmap((void *) p, len) != 0) { + atomic_store_explicit(&arg.go, 1, memory_order_release); + pthread_join(worker, NULL); + sigaction(SIGSEGV, &old_sa, NULL); + return "munmap"; + } + + /* No syscall may intervene here: the sibling must observe EL1's + * inner-shareable invalidation before host metadata removal. + */ + atomic_store_explicit(&arg.go, 1, memory_order_release); + int result; + while (!(result = atomic_load_explicit(&arg.result, memory_order_acquire))) + spin_hint(); + pthread_join(worker, NULL); + sigaction(SIGSEGV, &old_sa, NULL); + return result < 0 ? "stale sibling translation survived TLBI" : NULL; +} + +static void test_l3_range_tlbi(void) +{ + TEST("RVALE1IS invalidates sibling L3 translation"); + const char *error = run_sibling_tlbi(16ULL << 10, 4096); + if (error) { + FAIL(error); + return; + } + PASS(); +} + +static void test_large_l2_range_tlbi(void) +{ + TEST("SCALE=3 RVALE1IS invalidates sibling L2 translation"); + const size_t len = 320ULL << 20; /* exceeds SCALE=2's 256MiB maximum */ + const char *error = run_sibling_tlbi(len, 2ULL << 20); + if (error) { + FAIL(error); + return; + } + PASS(); +} + +typedef struct { + _Atomic uintptr_t ptr; + _Atomic int ready; + _Atomic int done; + _Atomic int release; +} handoff_arg_t; + +static inline void spin_hint(void) +{ + __asm__ volatile("yield" ::: "memory"); +} + +static void *handoff_worker(void *opaque) +{ + handoff_arg_t *arg = opaque; + uint8_t *warm = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (warm == MAP_FAILED) { + atomic_store_explicit(&arg->done, -1, memory_order_release); + atomic_store_explicit(&arg->ready, 1, memory_order_release); + return NULL; + } + warm[0] = 1; + atomic_store_explicit(&arg->ready, 1, memory_order_release); + uintptr_t ptr; + while (!(ptr = atomic_load_explicit(&arg->ptr, memory_order_acquire))) + spin_hint(); + int rc = munmap((void *) ptr, 4096); + atomic_store_explicit(&arg->done, rc == 0 ? 1 : -1, memory_order_release); + while (!atomic_load_explicit(&arg->release, memory_order_acquire)) + spin_hint(); + munmap(warm, 4096); + return NULL; +} + +static void test_cross_vcpu_handoff(void) +{ + TEST("cross-vCPU mmap publication then munmap retirement"); + struct sigaction sa = {.sa_handler = segv_handler}; + struct sigaction old_sa; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGSEGV, &sa, &old_sa) != 0) { + FAIL("sigaction"); + return; + } + + handoff_arg_t arg = {0}; + pthread_t worker; + if (pthread_create(&worker, NULL, handoff_worker, &arg) != 0) { + FAIL("pthread_create"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + while (!atomic_load_explicit(&arg.ready, memory_order_acquire)) + spin_hint(); + + /* Keep the worker alive after munmap so its next thread-exit syscall cannot + * drain either ring. The fault below is the first natural VM exit after A's + * mmap publication and B's retirement. + */ + uint8_t *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + atomic_store_explicit(&arg.release, 1, memory_order_release); + pthread_join(worker, NULL); + FAIL("mmap"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + atomic_store_explicit(&arg.ptr, (uintptr_t) p, memory_order_release); + int done; + while (!(done = atomic_load_explicit(&arg.done, memory_order_acquire))) + spin_hint(); + + int faulted = 0; + if (done > 0 && sigsetjmp(segv_jmp, 1) == 0) + (void) *(volatile uint8_t *) p; + else if (done > 0) + faulted = 1; + + atomic_store_explicit(&arg.release, 1, memory_order_release); + pthread_join(worker, NULL); + sigaction(SIGSEGV, &old_sa, NULL); + if (done < 0) { + FAIL("worker munmap"); + return; + } + if (!faulted) { + FAIL("retired cross-vCPU mapping remained accessible"); + return; + } + PASS(); +} + +static void test_mixed_size_recycled_va_reads_zero(void) +{ + TEST("recycled mixed-size VA reads zero"); + static const size_t sizes[] = { + 64ULL << 10, 3ULL << 20, 20ULL << 10, 1ULL << 20, + 5ULL << 20, 96ULL << 10, 2ULL << 20, 512ULL << 10, + }; + + for (int i = 0; i < 128; i++) { + size_t len = sizes[i % (int) (sizeof(sizes) / sizeof(sizes[0]))]; + volatile uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + if (p[0] != 0 || p[len / 2] != 0 || p[len - 1] != 0) { + munmap((void *) p, len); + FAIL("recycled VA exposed stale bytes"); + return; + } + p[0] = (uint8_t) (i + 1); + p[len / 2] = (uint8_t) (i ^ 0x5a); + p[len - 1] = (uint8_t) (i ^ 0xa5); + if (munmap((void *) p, len) != 0) { + FAIL("munmap"); + return; + } + } + PASS(); +} + +static void test_repeated_munmap_does_not_alias(void) +{ + TEST("repeated munmap does not alias two live mappings"); + const size_t len = 64ULL << 10; + volatile uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("initial mmap"); + return; + } + p[0] = 1; + if (munmap((void *) p, len) != 0 || munmap((void *) p, len) != 0) { + FAIL("repeated munmap"); + return; + } + + /* This fault drains both retire records. Only the first still has region + * coverage, so only it may return VA to the allocator; the second must not + * hand the same extent out a second time. + */ + volatile uint8_t *bridge = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (bridge == MAP_FAILED) { + FAIL("bridge mmap"); + return; + } + bridge[0] = 2; + + volatile uint8_t *a = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + volatile uint8_t *b = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (a == MAP_FAILED || b == MAP_FAILED || a == b) { + if (a != MAP_FAILED) + munmap((void *) a, len); + if (b != MAP_FAILED && b != a) + munmap((void *) b, len); + munmap((void *) bridge, len); + FAIL("duplicate extent allocation"); + return; + } + a[0] = 0x31; + b[0] = 0x42; + if (a[0] != 0x31 || b[0] != 0x42) { + FAIL("distinct mappings aliased"); + return; + } + munmap((void *) a, len); + munmap((void *) b, len); + munmap((void *) bridge, len); + PASS(); +} + +typedef struct { + _Atomic uintptr_t ptr; + _Atomic int state; + _Atomic int release; +} observer_arg_t; + +enum { + OBSERVER_STARTING, + OBSERVER_MAPPED, + OBSERVER_UNMAP_REQUESTED, + OBSERVER_UNMAPPED, + OBSERVER_MMAP_FAILED = -1, + OBSERVER_MUNMAP_FAILED = -2, +}; + +static void *observer_worker(void *opaque) +{ + observer_arg_t *arg = opaque; + void *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + atomic_store_explicit(&arg->state, OBSERVER_MMAP_FAILED, + memory_order_release); + return NULL; + } + + atomic_store_explicit(&arg->ptr, (uintptr_t) p, memory_order_relaxed); + atomic_store_explicit(&arg->state, OBSERVER_MAPPED, memory_order_release); + while (atomic_load_explicit(&arg->state, memory_order_acquire) == + OBSERVER_MAPPED) + spin_hint(); + + if (munmap(p, 4096) != 0) { + atomic_store_explicit(&arg->state, OBSERVER_MUNMAP_FAILED, + memory_order_release); + return NULL; + } + atomic_store_explicit(&arg->state, OBSERVER_UNMAPPED, memory_order_release); + while (!atomic_load_explicit(&arg->release, memory_order_acquire)) + spin_hint(); + return NULL; +} + +static int observer_wait_for_state(observer_arg_t *arg, int wanted) +{ + int state; + do { + state = atomic_load_explicit(&arg->state, memory_order_acquire); + spin_hint(); + } while (state >= 0 && state != wanted); + return state; +} + +static void observer_release(observer_arg_t *arg, pthread_t worker) +{ + atomic_store_explicit(&arg->release, 1, memory_order_release); + pthread_join(worker, NULL); +} + +static void test_msync_observes_fastpath(void) +{ + TEST("msync observes sibling fast mmap/munmap"); + observer_arg_t arg = {0}; + pthread_t worker; + if (pthread_create(&worker, NULL, observer_worker, &arg) != 0) { + FAIL("pthread_create"); + return; + } + if (observer_wait_for_state(&arg, OBSERVER_MAPPED) != OBSERVER_MAPPED) { + pthread_join(worker, NULL); + FAIL("worker mmap"); + return; + } + + void *p = (void *) atomic_load_explicit(&arg.ptr, memory_order_relaxed); + bool mapped_visible = msync(p, 4096, MS_SYNC) == 0; + atomic_store_explicit(&arg.state, OBSERVER_UNMAP_REQUESTED, + memory_order_release); + int state = observer_wait_for_state(&arg, OBSERVER_UNMAPPED); + errno = 0; + bool unmapped_visible = state == OBSERVER_UNMAPPED && + msync(p, 4096, MS_SYNC) == -1 && errno == ENOMEM; + + observer_release(&arg, worker); + if (!mapped_visible || !unmapped_visible) { + FAIL("msync used stale fast-path VMA metadata"); + return; + } + PASS(); +} + +static bool proc_paths_contain(uintptr_t address, bool expected) +{ + char pid_maps[64], pid_smaps[64]; + snprintf(pid_maps, sizeof(pid_maps), "/proc/%ld/maps", (long) getpid()); + snprintf(pid_smaps, sizeof(pid_smaps), "/proc/%ld/smaps", (long) getpid()); + const char *paths[] = { + "/proc/self/maps", + pid_maps, + "/proc/self/smaps", + pid_smaps, + }; + + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { + uintptr_t lo = 0, hi = 0; + int found = proc_extent_for(paths[i], address, &lo, &hi); + if (found < 0 || (found == 1) != expected) + return false; + } + return true; +} + +static void test_proc_observes_fastpath(void) +{ + TEST("maps/smaps observe sibling fast mmap/munmap"); + observer_arg_t arg = {0}; + pthread_t worker; + if (pthread_create(&worker, NULL, observer_worker, &arg) != 0) { + FAIL("pthread_create"); + return; + } + if (observer_wait_for_state(&arg, OBSERVER_MAPPED) != OBSERVER_MAPPED) { + pthread_join(worker, NULL); + FAIL("worker mmap"); + return; + } + + uintptr_t p = atomic_load_explicit(&arg.ptr, memory_order_relaxed); + bool mapped_visible = proc_paths_contain(p, true); + atomic_store_explicit(&arg.state, OBSERVER_UNMAP_REQUESTED, + memory_order_release); + int state = observer_wait_for_state(&arg, OBSERVER_UNMAPPED); + bool unmapped_visible = + state == OBSERVER_UNMAPPED && proc_paths_contain(p, false); + + observer_release(&arg, worker); + if (!mapped_visible || !unmapped_visible) { + FAIL("maps/smaps used stale fast-path VMA metadata"); + return; + } + PASS(); +} + +typedef struct { + _Atomic int ready; + _Atomic int go; + _Atomic int stop; + _Atomic int failed; + _Atomic unsigned churned; +} wx_churn_arg_t; + +static void *wx_churn_worker(void *opaque) +{ + wx_churn_arg_t *arg = opaque; + atomic_fetch_add_explicit(&arg->ready, 1, memory_order_release); + while (!atomic_load_explicit(&arg->go, memory_order_acquire)) + spin_hint(); + + while (!atomic_load_explicit(&arg->stop, memory_order_acquire)) { + volatile uint8_t *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + atomic_store_explicit(&arg->failed, 1, memory_order_release); + break; + } + p[0] = 0x5a; + if (munmap((void *) p, 4096) != 0) { + atomic_store_explicit(&arg->failed, 1, memory_order_release); + break; + } + atomic_fetch_add_explicit(&arg->churned, 1, memory_order_release); + } + return NULL; +} + +static void test_wx_toggle_during_fastpath_churn(void) +{ + TEST("W^X toggle during sibling fast mmap churn"); + enum { NTHREADS = 4, NCYCLES = 256 }; + pthread_t workers[NTHREADS]; + wx_churn_arg_t arg = {0}; + int made = 0; + for (; made < NTHREADS; made++) { + if (pthread_create(&workers[made], NULL, wx_churn_worker, &arg) != 0) + break; + } + if (made != NTHREADS) { + atomic_store_explicit(&arg.go, 1, memory_order_release); + atomic_store_explicit(&arg.stop, 1, memory_order_release); + for (int i = 0; i < made; i++) + pthread_join(workers[i], NULL); + FAIL("pthread_create"); + return; + } + + while (atomic_load_explicit(&arg.ready, memory_order_acquire) != NTHREADS) + spin_hint(); + + uint32_t *code = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + bool ok = code != MAP_FAILED; + if (ok) { + code[0] = 0x52800020u; /* mov w0, #1 */ + code[1] = 0xd65f03c0u; /* ret */ + __builtin___clear_cache((char *) code, (char *) (code + 2)); + uint32_t (*fn)(void) = (uint32_t (*)(void)) code; + ok = mprotect(code, 4096, PROT_READ | PROT_WRITE | PROT_EXEC) == 0 && + fn() == 1; + + atomic_store_explicit(&arg.go, 1, memory_order_release); + while (!atomic_load_explicit(&arg.failed, memory_order_acquire) && + atomic_load_explicit(&arg.churned, memory_order_acquire) < 64) + spin_hint(); + + for (uint32_t i = 0; i < NCYCLES && ok; i++) { + uint32_t imm = (i % 0xfffu) + 1; + code[0] = 0x52800000u | (imm << 5); + code[1] = 0xd65f03c0u; + __builtin___clear_cache((char *) code, (char *) (code + 2)); + if (fn() != imm) + ok = false; + } + } else { + atomic_store_explicit(&arg.go, 1, memory_order_release); + } + + atomic_store_explicit(&arg.stop, 1, memory_order_release); + for (int i = 0; i < NTHREADS; i++) + pthread_join(workers[i], NULL); + if (code != MAP_FAILED) + munmap(code, 4096); + + if (!ok || atomic_load_explicit(&arg.failed, memory_order_acquire) || + atomic_load_explicit(&arg.churned, memory_order_acquire) < 64) { + FAIL("W^X toggle or mmap churn failed"); + return; + } + PASS(); +} + +typedef struct { + int iterations; + _Atomic int *failed; +} storm_arg_t; + +static void *storm_worker(void *opaque) +{ + storm_arg_t *arg = opaque; + for (int i = 0; i < arg->iterations; i++) { + size_t len = (size_t) ((i & 7) + 1) * 4096; + uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + __atomic_store_n(arg->failed, 1, __ATOMIC_RELAXED); + break; + } + p[0] = (uint8_t) i; + p[len - 1] = (uint8_t) (i ^ 0x5a); + if (munmap(p, len) != 0) { + __atomic_store_n(arg->failed, 1, __ATOMIC_RELAXED); + break; + } + } + return NULL; +} + +static void test_mt_storm_and_fork_exec(void) +{ + TEST("multi-vCPU mmap storm with fork+exec revocation"); + enum { NTHREADS = 8 }; + pthread_t threads[NTHREADS]; + _Atomic int failed = 0; + storm_arg_t arg = {.iterations = 400, .failed = &failed}; + + int made = 0; + for (; made < NTHREADS; made++) { + if (pthread_create(&threads[made], NULL, storm_worker, &arg) != 0) { + __atomic_store_n(&failed, 1, __ATOMIC_RELAXED); + break; + } + } + + pid_t pid = fork(); + if (pid == 0) { + char *const argv[] = {(char *) "/proc/self/exe", NULL}; + char *const envp[] = {(char *) "ELFUSE_FASTPATH_EXEC_CHILD=1", NULL}; + execve(argv[0], argv, envp); + _exit(111); + } + if (pid < 0) + __atomic_store_n(&failed, 1, __ATOMIC_RELAXED); + + for (int i = 0; i < made; i++) + pthread_join(threads[i], NULL); + + if (pid > 0) { + int status = 0; + if (waitpid(pid, &status, 0) != pid || !WIFEXITED(status) || + WEXITSTATUS(status) != 0) + __atomic_store_n(&failed, 1, __ATOMIC_RELAXED); + } + + /* The parent's arenas were revoked for the fork snapshot. This pair makes + * the first call take the generation fallback and verifies service resumes. + */ + uint8_t *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + __atomic_store_n(&failed, 1, __ATOMIC_RELAXED); + else { + p[0] = 7; + munmap(p, 4096); + } + + if (__atomic_load_n(&failed, __ATOMIC_RELAXED)) { + FAIL("storm/fork/exec worker failure"); + return; + } + PASS(); +} + +static int stats_stream(size_t len, int iterations, bool release_each) +{ + for (int i = 0; i < iterations; i++) { + void *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + if (release_each && munmap(p, len) != 0) + return 1; + } + return 0; +} + +static int stats_mixed_churn(void) +{ + static const size_t sizes[] = { + 64ULL << 10, 3ULL << 20, 20ULL << 10, 1ULL << 20, + 5ULL << 20, 96ULL << 10, 2ULL << 20, 512ULL << 10, + }; + for (int i = 0; i < 160; i++) { + size_t len = sizes[i % (int) (sizeof(sizes) / sizeof(sizes[0]))]; + volatile uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + if (p[0] != 0 || p[len - 1] != 0) + return 1; + p[0] = 1; + p[len - 1] = 2; + if (munmap((void *) p, len) != 0) + return 1; + } + return 0; +} + +static int stats_large_materialized(void) +{ + const size_t len = 32ULL << 30; + volatile uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + p[0] = 1; + return munmap((void *) p, len) != 0; +} + +static int stats_fork_no_topup(void) +{ + /* Leave exactly half of the initial 64 MiB arena free. The 32 MiB + * registration becomes the low-water mark, so the ordinary mmap lock + * acquire would speculatively refill immediately before fork revokes all + * arenas. + */ + void *p = mmap(NULL, 32ULL << 20, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + + pid_t pid = fork(); + if (pid == 0) + _exit(0); + if (pid < 0) + return 1; + + int status = 0; + if (waitpid(pid, &status, 0) != pid || !WIFEXITED(status) || + WEXITSTATUS(status) != 0) + return 1; + return 0; +} + +static int stats_invalid_futex(void) +{ + uint32_t *words = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (words == MAP_FAILED) + return 1; + + long rc = raw_syscall6(__NR_futex, (long) words, 127, 0, 0, 0, 0); + if (rc != -38) + return 1; + rc = raw_syscall6(__NR_futex, (long) words, FUTEX_WAIT_BITSET, 0, 0, 0, 0); + if (rc != -22) + return 1; + rc = raw_syscall6(__NR_futex, (long) words, FUTEX_CMP_REQUEUE, 0, 0, + (long) ((uint8_t *) words + 1), 0); + if (rc != -22) + return 1; + rc = raw_syscall6(__NR_futex, (long) words, FUTEX_WAKE_OP, 0, 0, + (long) (words + 1), 5L << 28); + if (rc != -38) + return 1; + + return munmap(words, 4096) != 0; +} + +static int run_observer_semantics(void) +{ + test_msync_observes_fastpath(); + test_wx_toggle_during_fastpath_churn(); + test_proc_observes_fastpath(); + return fails != 0; +} + +static int run_stats_case(const char *name) +{ + if (strcmp(name, "observer-semantics") == 0) + return run_observer_semantics(); + if (strcmp(name, "ring-full") == 0) + return stats_stream(64ULL << 10, 40, false); + if (strcmp(name, "np2-10m") == 0) + return stats_stream(10ULL << 20, 96, false); + if (strcmp(name, "np2-48m") == 0) + return stats_stream(48ULL << 20, 48, false); + if (strcmp(name, "np2-100m") == 0) + return stats_stream(100ULL << 20, 30, false); + if (strcmp(name, "escalation") == 0) { + if (stats_stream(10ULL << 20, 48, false) != 0) + return 1; + return stats_stream(512ULL << 20, 3, false); + } + if (strcmp(name, "giant-guard") == 0) { + if (stats_stream(10ULL << 20, 32, false) != 0) + return 1; + for (int i = 0; i < 6; i++) { + if (stats_stream(2ULL << 30, 1, false) != 0 || + stats_stream(10ULL << 20, 1, false) != 0) + return 1; + } + return 0; + } + if (strcmp(name, "adaptive-small") == 0) + return stats_stream(64ULL << 10, 1100, false); + if (strcmp(name, "adaptive-retention") == 0) { + if (stats_stream(64ULL << 10, 1100, false) != 0) + return 1; + + /* The first request selects a 16GiB arena, the next 32 consume it, and + * the last forces a capacity rollover that must retain the target. + */ + return stats_stream(500ULL << 20, 34, false); + } + if (strcmp(name, "adaptive-rewind-growth") == 0) + + /* The first 64MiB arena holds eight 8MiB mappings. The ninth mmap takes + * the capacity fallback after the matching munmaps let host drain + * rewind the arena; refill must grow it to the 32-entry target instead + * of retaining an arena that will miss every eight calls. + */ + return stats_stream(8ULL << 20, 41, true); + if (strcmp(name, "recycle") == 0) + return stats_stream(64ULL << 10, 6000, true); + if (strcmp(name, "mixed-churn") == 0) + return stats_mixed_churn(); + if (strcmp(name, "large-materialized") == 0) + return stats_large_materialized(); + if (strcmp(name, "fork-no-topup") == 0) + return stats_fork_no_topup(); + if (strcmp(name, "invalid-futex") == 0) + return stats_invalid_futex(); + return 2; +} + +int main(int argc, char **argv) +{ + if (argc == 2 && strcmp(argv[1], "--observer-semantics") == 0) + return run_observer_semantics(); + if (argc == 2 && strncmp(argv[1], "--stats-", 8) == 0) + return run_stats_case(argv[1] + 8); + + if (getenv("ELFUSE_FASTPATH_EXEC_CHILD")) { + uint8_t *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + p[0] = 0xa5; + return p[0] == 0xa5 ? 0 : 1; + } + + test_fidelity(); + test_exhaustion_fallback(); + test_l3_range_tlbi(); + test_large_l2_range_tlbi(); + test_cross_vcpu_handoff(); + test_mixed_size_recycled_va_reads_zero(); + test_repeated_munmap_does_not_alias(); + run_observer_semantics(); + test_mt_storm_and_fork_exec(); + + printf("\ntest-mmap-fastpath: %d passed, %d failed - %s\n", passes, fails, + fails ? "FAIL" : "PASS"); + return fails ? 1 : 0; +} diff --git a/tests/test-mmap-lazy.c b/tests/test-mmap-lazy.c new file mode 100644 index 00000000..5d86305d --- /dev/null +++ b/tests/test-mmap-lazy.c @@ -0,0 +1,932 @@ +/* + * Lazy anonymous mmap regression tests + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Private anonymous mappings defer page-table creation and zeroing to first + * touch. These tests pin down the guest-visible contract of that laziness: huge + * reservations succeed and read as zeros, address reuse never leaks stale + * bytes, host-side syscall access (read/write/futex) works on memory the guest + * never touched, PROT_NONE stays a faulting reservation, data survives + * PROT_NONE round trips and fork, and concurrent first touch from multiple + * threads never loses a write to the deferred zeroing. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" + +int passes = 0, fails = 0; + +#define BLOCK_2MIB (2ULL << 20) + +#ifndef FUTEX_WAIT +#define FUTEX_WAIT 0 +#define FUTEX_WAKE 1 +#endif + +/* Largest plain anonymous RW mapping the kernel grants. On elfuse the lazy path + * must take this well past physical memory; on real Linux the result depends on + * the overcommit heuristic, so the tests only require >= 1 GiB and probe + * downward. + */ +static void *map_largest(size_t *out_size) +{ + static const size_t sizes[] = { + 64ULL << 30, + 16ULL << 30, + 4ULL << 30, + 1ULL << 30, + }; + for (unsigned i = 0; i < sizeof(sizes) / sizeof(sizes[0]); i++) { + void *p = mmap(NULL, sizes[i], PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p != MAP_FAILED) { + *out_size = sizes[i]; + return p; + } + } + return NULL; +} + +static void test_huge_sparse(void) +{ + TEST("huge mmap + sparse touch"); + size_t size = 0; + volatile uint8_t *p = map_largest(&size); + if (!p || size < (1ULL << 30)) { + FAIL("no >=1GiB anonymous mapping granted"); + return; + } + + /* Sparse probes: start, one per size/8 stride, last page. All must read + * zero and accept writes. + */ + for (unsigned i = 0; i < 8; i++) { + size_t off = (size / 8) * i; + if (p[off] != 0) { + FAIL("fresh mapping reads nonzero"); + munmap((void *) p, size); + return; + } + p[off] = (uint8_t) (i + 1); + } + if (p[size - 1] != 0) { + FAIL("last page reads nonzero"); + munmap((void *) p, size); + return; + } + for (unsigned i = 0; i < 8; i++) { + size_t off = (size / 8) * i; + if (p[off] != (uint8_t) (i + 1)) { + FAIL("sparse write lost"); + munmap((void *) p, size); + return; + } + } + if (munmap((void *) p, size) != 0) { + FAIL("munmap"); + return; + } + PASS(); +} + +static void test_zero_reuse(void) +{ + TEST("address reuse reads zero"); + size_t size = 4ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap 1"); + return; + } + memset(p, 0xa5, size); + munmap(p, size); + uint8_t *q = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q == MAP_FAILED) { + FAIL("mmap 2"); + return; + } + + /* The allocator typically reuses the freed range; either way no byte may be + * nonzero. Check one page per 2MiB block plus both ends. + */ + for (size_t off = 0; off < size; off += 4096) { + if (q[off] != 0) { + FAIL("stale data after reuse"); + munmap(q, size); + return; + } + } + munmap(q, size); + PASS(); +} + +static void test_hinted_tail_zero(void) +{ + TEST("hinted tail mmap reads zero"); + size_t size = 2ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap base"); + return; + } + p[0] = 0x3c; + p[size - 1] = 0xc3; + + uint8_t *hint = p + size; + uint8_t *q = mmap(hint, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q == MAP_FAILED) { + FAIL("mmap hint"); + munmap(p, size); + return; + } + if (q[0] != 0 || q[size - 1] != 0 || p[0] != 0x3c || p[size - 1] != 0xc3) { + FAIL("hinted tail leaked stale bytes or clobbered neighbor"); + munmap(q, size); + munmap(p, size); + return; + } + munmap(q, size); + munmap(p, size); + PASS(); +} + +static void test_partial_block_reuse(void) +{ + TEST("partial-block reuse preserves neighbor"); + const size_t half = BLOCK_2MIB / 2; + uint8_t *p = mmap(NULL, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + p[17] = 0xa5; + p[half + 17] = 0x5a; + if (mprotect(p + half, half, PROT_READ) != 0 || munmap(p, half) != 0) { + FAIL("split/unmap"); + munmap(p, BLOCK_2MIB); + return; + } + + uint8_t *q = mmap(p, half, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q == MAP_FAILED || q != p) { + FAIL("freed half was not reused at hint"); + if (q != MAP_FAILED) + munmap(q, half); + munmap(p + half, half); + return; + } + if (q[17] != 0 || q[half - 1] != 0 || p[half + 17] != 0x5a) { + FAIL("partial zero clobbered neighbor or leaked stale data"); + munmap(q, half); + munmap(p + half, half); + return; + } + munmap(q, half); + munmap(p + half, half); + PASS(); +} + +static uint8_t *map_clean_block_reuse(void) +{ + uint8_t *p = mmap(NULL, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return MAP_FAILED; + memset(p, 0xcc, BLOCK_2MIB); + if (munmap(p, BLOCK_2MIB) != 0) { + munmap(p, BLOCK_2MIB); + return MAP_FAILED; + } + + uint8_t *q = mmap(p, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q != p) { + if (q != MAP_FAILED) + munmap(q, BLOCK_2MIB); + return MAP_FAILED; + } + return q; +} + +static void test_file_overlay_reuse(void) +{ + TEST("file overlay teardown then lazy reuse"); + char path[] = "/tmp/elfuse-dirty-map.XXXXXX"; + int fd = mkstemp(path); + if (fd < 0) { + FAIL("mkstemp"); + return; + } + unlink(path); + if (ftruncate(fd, BLOCK_2MIB) != 0) { + FAIL("ftruncate"); + close(fd); + return; + } + uint8_t first = 0xa7, last = 0x5c; + if (pwrite(fd, &first, 1, 17) != 1 || + pwrite(fd, &last, 1, BLOCK_2MIB - 1) != 1) { + FAIL("pwrite"); + close(fd); + return; + } + + uint8_t *reserve = mmap(NULL, 2 * BLOCK_2MIB, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reserve == MAP_FAILED) { + FAIL("reserve"); + close(fd); + return; + } + uintptr_t aligned = + ((uintptr_t) reserve + BLOCK_2MIB - 1) & ~(BLOCK_2MIB - 1); + munmap(reserve, 2 * BLOCK_2MIB); + uint8_t *target = (uint8_t *) aligned; + + uint8_t *file = mmap(target, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, fd, 0); + if (file != target || file[17] != first || file[BLOCK_2MIB - 1] != last) { + FAIL("file mmap"); + if (file != MAP_FAILED) + munmap(file, BLOCK_2MIB); + close(fd); + return; + } + file[BLOCK_2MIB / 2] = 0xe1; + if (munmap(file, BLOCK_2MIB) != 0) { + FAIL("file munmap"); + close(fd); + return; + } + close(fd); + + uint8_t *anon = mmap(target, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (anon != target || anon[17] != 0 || anon[BLOCK_2MIB / 2] != 0 || + anon[BLOCK_2MIB - 1] != 0) { + FAIL("stale file bytes after lazy reuse"); + if (anon != MAP_FAILED) + munmap(anon, BLOCK_2MIB); + return; + } + munmap(anon, BLOCK_2MIB); + PASS(); +} + +static void test_read_into_lazy(void) +{ + TEST("read() into untouched mapping"); + size_t size = 6ULL << 20; + uint8_t *buf = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int fds[2]; + if (buf == MAP_FAILED || pipe(fds) != 0) { + FAIL("setup"); + return; + } + static const char msg[] = "lazy-host-access-payload"; + /* Unaligned target crossing into the mapping's third 2MiB block. */ + size_t off = (4ULL << 20) + 123; + if (write(fds[1], msg, sizeof(msg)) != (ssize_t) sizeof(msg) || + read(fds[0], buf + off, sizeof(msg)) != (ssize_t) sizeof(msg)) { + FAIL("pipe copy through untouched buffer"); + goto out; + } + if (memcmp(buf + off, msg, sizeof(msg)) != 0) { + FAIL("payload corrupted"); + goto out; + } + + /* A guest touch elsewhere in the same 2MiB block must not re-zero the + * host-written payload (deferred-zeroing idempotence). + */ + buf[(4ULL << 20) + 64 * 1024] = 7; + if (memcmp(buf + off, msg, sizeof(msg)) != 0) { + FAIL("payload clobbered by later fault in same block"); + goto out; + } + /* Untouched parts of the mapping still read zero. */ + for (size_t i = 0; i < 4096; i++) { + if (buf[i] != 0) { + FAIL("nonzero byte in untouched block"); + goto out; + } + } + PASS(); +out: + close(fds[0]); + close(fds[1]); + munmap(buf, size); +} + +static void test_write_from_lazy(void) +{ + TEST("write() from untouched mapping"); + size_t size = 2ULL << 20; + uint8_t *src = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int fds[2]; + if (src == MAP_FAILED || pipe(fds) != 0) { + FAIL("setup"); + return; + } + uint8_t back[512]; + memset(back, 0xff, sizeof(back)); + if (write(fds[1], src + 4096, sizeof(back)) != (ssize_t) sizeof(back) || + read(fds[0], back, sizeof(back)) != (ssize_t) sizeof(back)) { + FAIL("pipe copy from untouched buffer"); + goto out; + } + for (size_t i = 0; i < sizeof(back); i++) { + if (back[i] != 0) { + FAIL("untouched buffer sent nonzero bytes"); + goto out; + } + } + PASS(); +out: + close(fds[0]); + close(fds[1]); + munmap(src, size); +} + +static void test_prot_none_roundtrip(void) +{ + TEST("mprotect NONE round trip keeps data"); + size_t size = 4ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + memset(p, 0x5c, 8192); + p[size - 1] = 0x77; + if (mprotect(p, size, PROT_NONE) != 0 || + mprotect(p, size, PROT_READ | PROT_WRITE) != 0) { + FAIL("mprotect"); + munmap(p, size); + return; + } + if (p[0] != 0x5c || p[8191] != 0x5c || p[size - 1] != 0x77 || + p[16384] != 0) { + FAIL("data lost or stale bytes after round trip"); + munmap(p, size); + return; + } + munmap(p, size); + PASS(); +} + +static void test_reserve_commit(void) +{ + TEST("PROT_NONE reserve + mprotect commit"); + size_t size = 1ULL << 30; + uint8_t *p = mmap(NULL, size, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + if (p == MAP_FAILED) { + FAIL("reserve"); + return; + } + uint8_t *slab = p + (512ULL << 20); + if (mprotect(slab, 8ULL << 20, PROT_READ | PROT_WRITE) != 0) { + FAIL("commit"); + munmap(p, size); + return; + } + for (size_t off = 0; off < (8ULL << 20); off += 4096) { + if (slab[off] != 0) { + FAIL("committed slab reads nonzero"); + munmap(p, size); + return; + } + } + slab[0] = 1; + slab[(8ULL << 20) - 1] = 2; + if (slab[0] != 1 || slab[(8ULL << 20) - 1] != 2) { + FAIL("committed slab write lost"); + munmap(p, size); + return; + } + munmap(p, size); + PASS(); +} + +static sigjmp_buf segv_jmp; + +static void segv_handler(int sig) +{ + (void) sig; + siglongjmp(segv_jmp, 1); +} + +static void test_prot_none_faults(void) +{ + TEST("PROT_NONE|NORESERVE still faults"); + size_t size = 16ULL << 20; + volatile uint8_t *p = + mmap(NULL, size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, + -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + struct sigaction sa = {0}, old_sa; + sa.sa_handler = segv_handler; + sigaction(SIGSEGV, &sa, &old_sa); + int faulted = 0; + if (sigsetjmp(segv_jmp, 1) == 0) { + (void) p[BLOCK_2MIB + 5]; + } else { + faulted = 1; + } + sigaction(SIGSEGV, &old_sa, NULL); + munmap((void *) p, size); + + /* A lazy materializer that ignores prot would silently hand the guest a + * readable zero page here instead of SIGSEGV. + */ + EXPECT_TRUE(faulted, "read from PROT_NONE reservation did not fault"); +} + +static void test_fork_lazy(void) +{ + TEST("fork with lazy and clean-block reused mappings"); + size_t size = 8ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + + uint8_t *clean = map_clean_block_reuse(); + if (clean == MAP_FAILED) { + FAIL("clean-block reuse"); + munmap(p, size); + return; + } + + memset(p, 0x42, 4096); /* touch only block 0 */ + pid_t pid = fork(); + if (pid == 0) { + /* Child: inherited data intact, untouched block reads zero and is + * privately writable. + */ + if (p[0] != 0x42 || p[4095] != 0x42) + _exit(1); + if (p[4ULL << 20] != 0) + _exit(2); + p[4ULL << 20] = 0x99; + if (p[(4ULL << 20) + 1] != 0) + _exit(3); + if (clean[0] != 0 || clean[BLOCK_2MIB - 1] != 0) + _exit(4); + clean[123] = 0x77; + if (clean[124] != 0) + _exit(5); + _exit(0); + } + int st = 0; + bool ok = pid >= 0 && waitpid(pid, &st, 0) == pid && WIFEXITED(st) && + WEXITSTATUS(st) == 0; + /* Parent: child's private write must not leak back. */ + ok = ok && p[4ULL << 20] == 0 && clean[123] == 0; + munmap(clean, BLOCK_2MIB); + munmap(p, size); + if (!ok) { + FAIL("fork changed lazy or clean-block reused memory"); + return; + } + PASS(); +} + +static void test_futex_untouched(void) +{ + TEST("futex on untouched mapping"); + size_t size = 4ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + uint32_t *word = (uint32_t *) (p + (2ULL << 20) + 256); + /* WAKE on never-touched memory: no waiters, must not fault. */ + long r = syscall(SYS_futex, word, FUTEX_WAKE, 1, NULL, NULL, 0); + if (r != 0) { + FAIL("FUTEX_WAKE on untouched word"); + munmap(p, size); + return; + } + /* WAIT with expected=1: the word reads as zero, so EAGAIN. */ + r = syscall(SYS_futex, word, FUTEX_WAIT, 1, NULL, NULL, 0); + if (!(r == -1 && errno == EAGAIN)) { + FAIL("FUTEX_WAIT did not read zero from untouched word"); + munmap(p, size); + return; + } + munmap(p, size); + PASS(); +} + +/* Concurrent first touch: every thread writes its own slot in the same fresh + * 2MiB block, racing the deferred zeroing. A materializer that re-zeros an + * already-populated block loses some slots. + */ +#define MT_THREADS 4 +#define MT_ITERS 64 + +typedef struct { + pthread_mutex_t mutex; + pthread_cond_t start; + pthread_cond_t done; + pthread_barrier_t touch; + uint8_t *base; + unsigned generation; + int completed; + bool stop; +} mt_state_t; + +typedef struct { + mt_state_t *state; + int idx; +} mt_arg_t; + +static bool mt_state_init(mt_state_t *state) +{ + memset(state, 0, sizeof(*state)); + if (pthread_mutex_init(&state->mutex, NULL) != 0) + return false; + if (pthread_cond_init(&state->start, NULL) != 0) { + pthread_mutex_destroy(&state->mutex); + return false; + } + if (pthread_cond_init(&state->done, NULL) != 0) { + pthread_cond_destroy(&state->start); + pthread_mutex_destroy(&state->mutex); + return false; + } + if (pthread_barrier_init(&state->touch, NULL, MT_THREADS) != 0) { + pthread_cond_destroy(&state->done); + pthread_cond_destroy(&state->start); + pthread_mutex_destroy(&state->mutex); + return false; + } + return true; +} + +static void mt_state_destroy(mt_state_t *state) +{ + pthread_barrier_destroy(&state->touch); + pthread_cond_destroy(&state->done); + pthread_cond_destroy(&state->start); + pthread_mutex_destroy(&state->mutex); +} + +static void *mt_touch(void *argp) +{ + mt_arg_t *a = argp; + mt_state_t *state = a->state; + unsigned seen = 0; + + pthread_mutex_lock(&state->mutex); + for (;;) { + while (!state->stop && seen == state->generation) + pthread_cond_wait(&state->start, &state->mutex); + if (state->stop) + break; + + uint8_t *base = state->base; + seen = state->generation; + pthread_mutex_unlock(&state->mutex); + + pthread_barrier_wait(&state->touch); + base[a->idx * 64] = (uint8_t) (a->idx + 1); + /* Also touch a private block so several materializations race. */ + base[BLOCK_2MIB * (unsigned) (a->idx + 1) + 17] = + (uint8_t) (0x10 + a->idx); + + pthread_mutex_lock(&state->mutex); + state->completed++; + if (state->completed == MT_THREADS) + pthread_cond_signal(&state->done); + } + pthread_mutex_unlock(&state->mutex); + return NULL; +} + +static void test_mt_first_touch(void) +{ + TEST("concurrent first touch"); + mt_state_t state; + if (!mt_state_init(&state)) { + FAIL("worker synchronization setup"); + return; + } + pthread_t th[MT_THREADS]; + mt_arg_t args[MT_THREADS]; + int created = 0; + bool ok = true; + + /* Reuse workers so all 64 rounds stress materialization rather than glibc + * thread-stack allocation and teardown. + */ + for (int i = 0; i < MT_THREADS; i++) { + args[i] = (mt_arg_t) {&state, i}; + if (pthread_create(&th[i], NULL, mt_touch, &args[i]) != 0) { + ok = false; + break; + } + created++; + } + + for (int iter = 0; ok && iter < MT_ITERS; iter++) { + size_t size = BLOCK_2MIB * (MT_THREADS + 2); + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + ok = false; + break; + } + + pthread_mutex_lock(&state.mutex); + state.base = p; + state.completed = 0; + state.generation++; + pthread_cond_broadcast(&state.start); + while (state.completed != MT_THREADS) + pthread_cond_wait(&state.done, &state.mutex); + pthread_mutex_unlock(&state.mutex); + + for (int i = 0; i < MT_THREADS; i++) { + if (p[i * 64] != (uint8_t) (i + 1) || + p[BLOCK_2MIB * (unsigned) (i + 1) + 17] != + (uint8_t) (0x10 + i)) { + ok = false; + break; + } + } + munmap(p, size); + } + + pthread_mutex_lock(&state.mutex); + state.stop = true; + pthread_cond_broadcast(&state.start); + pthread_mutex_unlock(&state.mutex); + for (int i = 0; i < created; i++) + pthread_join(th[i], NULL); + mt_state_destroy(&state); + + if (!ok) { + FAIL("setup failed or a write was lost to materialization"); + return; + } + PASS(); +} + +typedef struct { + uint8_t *base; + size_t half; + int idx; + pthread_barrier_t *barrier; + int *error; +} claim_race_arg_t; + +static void *claim_race_touch(void *argp) +{ + claim_race_arg_t *a = argp; + pthread_barrier_wait(a->barrier); + a->base[64 * (unsigned) a->idx] = (uint8_t) (a->idx + 1); + return NULL; +} + +static void *claim_race_mutate(void *argp) +{ + claim_race_arg_t *a = argp; + uint8_t *neighbor = a->base + a->half; + uint8_t *hole = neighbor + 4096; + pthread_barrier_wait(a->barrier); + for (int i = 0; i < 16; i++) { + if (mprotect(neighbor, a->half, PROT_NONE) != 0 || + mprotect(neighbor, a->half, PROT_READ) != 0 || + munmap(hole, 4096) != 0) { + *a->error = 1; + return NULL; + } + void *r = mmap(hole, 4096, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (r != hole) { + *a->error = 1; + return NULL; + } + } + return NULL; +} + +static void test_claim_mutation_race(void) +{ + TEST("first-touch claim vs adjacent mutations"); + const size_t half = BLOCK_2MIB / 2; + uint8_t *p = mmap(NULL, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + p[half + 17] = 0x6d; + if (mprotect(p + half, half, PROT_READ) != 0 || munmap(p, half) != 0) { + FAIL("split/unmap"); + munmap(p, BLOCK_2MIB); + return; + } + uint8_t *q = mmap(p, half, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q != p) { + FAIL("reuse"); + if (q != MAP_FAILED) + munmap(q, half); + munmap(p + half, half); + return; + } + + pthread_barrier_t barrier; + pthread_barrier_init(&barrier, NULL, MT_THREADS + 1); + pthread_t workers[MT_THREADS], mutator; + claim_race_arg_t args[MT_THREADS + 1]; + int error = 0; + for (int i = 0; i < MT_THREADS; i++) { + args[i] = (claim_race_arg_t) {q, half, i, &barrier, &error}; + pthread_create(&workers[i], NULL, claim_race_touch, &args[i]); + } + args[MT_THREADS] = (claim_race_arg_t) {q, half, 0, &barrier, &error}; + pthread_create(&mutator, NULL, claim_race_mutate, &args[MT_THREADS]); + for (int i = 0; i < MT_THREADS; i++) + pthread_join(workers[i], NULL); + pthread_join(mutator, NULL); + pthread_barrier_destroy(&barrier); + + for (int i = 0; i < MT_THREADS; i++) { + if (q[64 * (unsigned) i] != (uint8_t) (i + 1)) + error = 1; + } + if (p[half + 17] != 0x6d) + error = 1; + munmap(q, half); + munmap(p + half, half); + if (error) { + FAIL("claim/mutation race corrupted data"); + return; + } + PASS(); +} + +static void test_adjacent_region_extension(void) +{ + TEST("adjacent fast-mmap region extension"); + enum { N_PAGES = 64 }; + uint8_t *pages[N_PAGES]; + int allocated = 0; + bool ok = true; + + for (int i = 0; i < N_PAGES; i++) { + pages[i] = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (pages[i] == MAP_FAILED) { + ok = false; + break; + } + allocated++; + pages[i][0] = (uint8_t) (i + 1); + } + for (int i = 0; ok && i < allocated; i++) { + if (pages[i][0] != (uint8_t) (i + 1)) + ok = false; + } + for (int i = 0; i < allocated; i++) + munmap(pages[i], 4096); + + if (!ok) { + FAIL("adjacent lazy mappings did not materialize independently"); + return; + } + PASS(); +} + +static void test_large_retire_reuse(void) +{ + TEST("large multi-block retire preserves neighbors and fork zeroes"); + const size_t body_len = 96ULL << 20; + const size_t total_len = body_len + (6ULL << 20); + uint8_t *p = mmap(NULL, total_len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + + uintptr_t aligned = + ((uintptr_t) p + BLOCK_2MIB * 2 - 1) & ~(uintptr_t) (BLOCK_2MIB - 1); + uint8_t *body = (uint8_t *) aligned; + size_t left_len = (size_t) (body - p); + size_t right_len = total_len - left_len - body_len; + if (left_len < BLOCK_2MIB || right_len < BLOCK_2MIB) { + FAIL("guard alignment"); + munmap(p, total_len); + return; + } + + memset(p, 0xa5, total_len); + if (munmap(body, body_len) != 0) { + FAIL("retire body"); + munmap(p, total_len); + return; + } + + /* MAP_FIXED is a metadata-reading slow path, so it must first drain the EL1 + * retirement. The 96 MiB dirty body spans many 2 MiB blocks, so this + * exercises retire-then-reuse at multi-block scale rather than the + * single-block case the smaller tests above already cover. + */ + uint8_t *q = mmap(body, body_len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (q != body) { + FAIL("fixed reuse"); + munmap(p, left_len); + munmap(body + body_len, right_len); + return; + } + + bool ok = p[0] == 0xa5 && body[-1] == 0xa5 && body[body_len] == 0xa5 && + p[total_len - 1] == 0xa5; + for (size_t off = 0; ok && off < body_len; off += 4096) + ok = q[off] == 0; + + pid_t pid = -1; + int st = 0; + if (ok) + pid = fork(); + if (pid == 0) { + for (size_t off = 0; off < body_len; off += BLOCK_2MIB) { + if (q[off] != 0) + _exit(1); + } + q[123] = 0x77; + _exit(q[124] == 0 ? 0 : 2); + } + if (pid < 0 || waitpid(pid, &st, 0) != pid || !WIFEXITED(st) || + WEXITSTATUS(st) != 0 || q[123] != 0) + ok = false; + + munmap(p, left_len); + munmap(q, body_len); + munmap(body + body_len, right_len); + if (!ok) { + FAIL("reuse leaked data, clobbered a neighbor, or broke fork"); + return; + } + PASS(); +} + +int main(void) +{ + test_huge_sparse(); + test_zero_reuse(); + test_hinted_tail_zero(); + test_partial_block_reuse(); + test_file_overlay_reuse(); + test_read_into_lazy(); + test_write_from_lazy(); + test_prot_none_roundtrip(); + test_reserve_commit(); + test_prot_none_faults(); + test_fork_lazy(); + test_futex_untouched(); + test_mt_first_touch(); + test_claim_mutation_race(); + test_adjacent_region_extension(); + test_large_retire_reuse(); + + SUMMARY("test-mmap-lazy"); + return fails ? 1 : 0; +} diff --git a/tests/test-mremap.c b/tests/test-mremap.c index dc97321b..bf8abe33 100644 --- a/tests/test-mremap.c +++ b/tests/test-mremap.c @@ -120,6 +120,106 @@ static void test_grow_maymove(void) munmap(q, 4096 * 4); } +static void test_grow_move_adjacent_fault(void) +{ + TEST("mremap fixed move keeps adjacent page unmapped"); + void *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap source"); + return; + } + char *dest = + mmap(NULL, 4096 * 3, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (dest == MAP_FAILED) { + FAIL("mmap dest"); + munmap(p, 4096); + return; + } + if (munmap(dest + 8192, 4096) < 0) { + FAIL("munmap guard"); + munmap(dest, 8192); + munmap(p, 4096); + return; + } + ((char *) p)[0] = 0x5a; + + void *q = mremap(p, 4096, 8192, MREMAP_MAYMOVE | MREMAP_FIXED, dest); + if (q == MAP_FAILED) { + FAIL("mremap move"); + munmap(dest, 8192); + munmap(p, 4096); + return; + } + if (q != dest || ((char *) q)[0] != 0x5a || ((char *) q)[4096] != 0) { + FAIL("mremap data"); + munmap(q, 8192); + return; + } + + pid_t pid = fork(); + if (pid == 0) { + volatile unsigned char value = *((volatile unsigned char *) q + 8192); + (void) value; + _exit(1); + } + int status = 0; + if (pid < 0 || waitpid(pid, &status, 0) != pid) { + FAIL("fork/wait"); + munmap(q, 8192); + return; + } + if ((WIFSIGNALED(status) && WTERMSIG(status) == SIGSEGV) || + (WIFEXITED(status) && WEXITSTATUS(status) == 139)) + PASS(); + else + FAIL("adjacent page became readable"); + munmap(q, 8192); +} + +static void test_fixed_preserves_neighbor_l3(void) +{ + TEST("mremap fixed preserves neighboring lazy PTEs"); + char *source = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (source == MAP_FAILED) { + FAIL("mmap source"); + return; + } + char *anchor = mmap(NULL, 8192, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (anchor == MAP_FAILED) { + FAIL("mmap anchor"); + munmap(source, 4096); + return; + } + char *dest = anchor + 4096; + if (munmap(dest, 4096) < 0) { + FAIL("munmap dest"); + munmap(anchor, 4096); + munmap(source, 4096); + return; + } + source[0] = 0x11; + anchor[0] = 0x7e; + + void *moved = + mremap(source, 4096, 4096, MREMAP_MAYMOVE | MREMAP_FIXED, dest); + if (moved == MAP_FAILED) { + FAIL("mremap fixed"); + munmap(anchor, 4096); + munmap(source, 4096); + return; + } + if (moved != dest || dest[0] != 0x11 || anchor[0] != 0x7e) + FAIL("neighboring lazy page changed"); + else + PASS(); + + munmap(dest, 4096); + munmap(anchor, 4096); +} + /* Test 3: grow without MAYMOVE fails if blocked */ static void test_grow_no_maymove(void) @@ -403,6 +503,8 @@ int main(void) test_shrink(); test_grow_maymove(); + test_grow_move_adjacent_fault(); + test_fixed_preserves_neighbor_l3(); test_grow_no_maymove(); test_fixed(); test_same_size(); diff --git a/tests/test-thread-churn.c b/tests/test-thread-churn.c index f76fc8e4..f1225b54 100644 --- a/tests/test-thread-churn.c +++ b/tests/test-thread-churn.c @@ -24,9 +24,13 @@ int passes = 0, fails = 0; -#define SEQUENTIAL_ROUNDS 150 +/* Keep this a slot-reuse regression rather than an HVF vCPU lifecycle soak. + * Eighty sequential workers exceed the 64-slot table by more than 16, and two + * 32-worker batches repeat reuse while unrelated workers remain live. + */ +#define SEQUENTIAL_ROUNDS 80 #define BATCH_SIZE 32 -#define BATCH_ROUNDS 6 +#define BATCH_ROUNDS 2 static void *churn_fn(void *arg) { @@ -40,7 +44,7 @@ static void *churn_fn(void *arg) */ static void test_sequential_churn(void) { - TEST("sequential churn (150 threads)"); + TEST("sequential churn (80 threads)"); for (int i = 0; i < SEQUENTIAL_ROUNDS; i++) { int ran = 0; @@ -67,7 +71,7 @@ static void test_sequential_churn(void) */ static void test_batch_churn(void) { - TEST("batch churn (6x32 threads)"); + TEST("batch churn (2x32 threads)"); for (int round = 0; round < BATCH_ROUNDS; round++) { pthread_t threads[BATCH_SIZE]; diff --git a/tests/test-tlbi-encoder-host.c b/tests/test-tlbi-encoder-host.c index 4757a2f0..d6f55226 100644 --- a/tests/test-tlbi-encoder-host.c +++ b/tests/test-tlbi-encoder-host.c @@ -47,12 +47,12 @@ static void check_field(const char *label, uint64_t got, uint64_t expect) /* Decompose the operand per ARM ARM D8.7.6 and compare each field against the * expected value. baseADDR is VA>>12 masked to 37 bits; TG must be 01 (4 KiB); - * SCALE must be 0; TTL must be 0; ASID must be 0. NUM derives from the page - * count via the ceil(pages/2) - 1 SCALE=0 encoding. + * TTL and ASID must be 0. NUM derives from the selected SCALE unit. */ static void verify_operand(uint64_t start_va, uint16_t pages, - uint64_t expect_num) + uint64_t expect_num, + uint64_t expect_scale) { uint64_t op = tlbi_rvae1is_operand(start_va, pages); @@ -76,7 +76,7 @@ static void verify_operand(uint64_t start_va, check_field(label, num, expect_num); snprintf(label, sizeof(label), "SCALE (pages=%u)", (unsigned) pages); - check_field(label, scale, 0); + check_field(label, scale, expect_scale); snprintf(label, sizeof(label), "TG (start=0x%llx)", (unsigned long long) start_va); @@ -100,30 +100,47 @@ int main(void) * pages 63 -> NUM 31 (covers 64) * pages 64 -> NUM 31 (covers 64) */ - verify_operand(0x10000000ULL, 2, 0); - verify_operand(0x10000000ULL, 3, 1); - verify_operand(0x10000000ULL, 16, 7); - verify_operand(0x10000000ULL, 17, 8); - verify_operand(0x10000000ULL, 32, 15); - verify_operand(0x10000000ULL, 63, 31); - verify_operand(0x10000000ULL, 64, 31); + verify_operand(0x10000000ULL, 2, 0, 0); + verify_operand(0x10000000ULL, 3, 1, 0); + verify_operand(0x10000000ULL, 16, 7, 0); + verify_operand(0x10000000ULL, 17, 8, 0); + verify_operand(0x10000000ULL, 32, 15, 0); + verify_operand(0x10000000ULL, 63, 31, 0); + verify_operand(0x10000000ULL, 64, 31, 0); + + /* SCALE=1 covers 64 pages per NUM step; SCALE=2 covers 2048. A lazy 2 MiB + * block is 512 pages and must therefore encode as SCALE=1, NUM=7. + */ + verify_operand(0x10000000ULL, 512, 7, 1); + verify_operand(0x10000000ULL, 2048, 31, 1); + verify_operand(0x10000000ULL, 8192, 3, 2); /* Boundary VAs. 4 KiB-aligned, low-VA, MMAP_BASE (8 GiB), high-VA just * below the 48-bit BaseADDR truncation point. */ - verify_operand(0x00000000ULL, 32, 15); /* zero base */ - verify_operand(0x200000000ULL, 32, 15); /* MMAP_BASE */ - verify_operand(0x800000000000ULL, 32, 15); /* Rosetta image */ - verify_operand(0x0000FFFFF0000000ULL, 32, 15); /* KBUF_USER_VA */ + verify_operand(0x00000000ULL, 32, 15, 0); /* zero base */ + verify_operand(0x200000000ULL, 32, 15, 0); /* MMAP_BASE */ + verify_operand(0x800000000000ULL, 32, 15, 0); /* Rosetta image */ + verify_operand(0x0000FFFFF0000000ULL, 32, 15, 0); /* KBUF_USER_VA */ /* Pathological inputs the clamp must catch: * pages = 0 -> clamped to 2 -> NUM 0 * pages = 1 -> clamped to 2 -> NUM 0 (callers never reach here) * pages = UINT16_MAX -> NUM clamped to 31 (saturating) */ - verify_operand(0x10000000ULL, 0, 0); - verify_operand(0x10000000ULL, 1, 0); - verify_operand(0x10000000ULL, UINT16_MAX, 31); + verify_operand(0x10000000ULL, 0, 0, 0); + verify_operand(0x10000000ULL, 1, 0, 0); + verify_operand(0x10000000ULL, UINT16_MAX, 31, 2); + + /* The accumulator widens a 2 MiB request to the SCALE=1 granule and keeps + * it on the single-shot RVAE path instead of degrading to broadcast. + */ + g_tlbi_range_supported = true; + tlbi_request_clear(); + tlbi_request_range(0x200000000ULL, 0x200200000ULL); + check_field("2MiB accumulator kind", cpu_tlbi_req.kind, TLBI_RANGE_LARGE); + check_field("2MiB accumulator pages", cpu_tlbi_req.pages, 512); + check_field("2MiB accumulator start", cpu_tlbi_req.start, 0x200000000ULL); /* TG bit is the architectural lynchpin -- if the encoder ever drops it the * integration tests on Apple Silicon would still pass. Pin a direct bit-46