perf(memtrack): decide ring buffer wakeups producer-side

Every submit called bpf_ringbuf_query(BPF_RB_AVAIL_DATA) to decide
whether to force a consumer wakeup. That reads the consumer position, a
cache line the polling thread on another CPU writes continuously, so
each event paid a cross-CPU miss for a decision that only changes once
per watermark.

Count submitted bytes per CPU instead and force a wakeup whenever the
watermark is crossed. Events are fixed size, so this is the same
cadence the query approximated, decided entirely on the producer side
with no shared cache line involved. A missing counter forces the wakeup
rather than risking a stalled consumer.

Measured on a malloc/free latency harness (p50 per pair, glibc):
2064 ns to 1102 ns, the largest of the three hot-path wins. Verified at
10M malloc/free pairs (20,006,217 events, ~800 MB through the 256 MB
ring buffer) with the dropped-event counter still at zero, so batched
wakeups keep up with a sustained high event rate.

Author: not-matthias

crates/memtrack/src/ebpf/c/utils/event_helpers.h

@@ -8,16 +8,36 @@
 BPF_RINGBUF(events, 256 * 1024 * 1024);
 BPF_ARRAY_MAP(dropped_events, __u64, 1);
 
-/* Wake the consumer only once this much unconsumed data has accumulated.
- * Per-event wakeups dominate submission cost at high event rates; batching
- * them behind a data watermark amortizes the wakeup to ~1 per thousand
- * events. The userspace poller's poll timeout flushes the tail that never
- * reaches the watermark. */
+/* Wake the consumer once this much data has been submitted. Per-event wakeups
+ * dominate submission cost at high event rates; batching them behind a data
+ * watermark amortizes the wakeup to ~1 per thousand events. The userspace
+ * poller's poll timeout flushes a tail that never reaches the watermark. */
 #define WAKEUP_DATA_SIZE (64 * 1024)
 
-static __always_inline long wake_flags(void) {
-    long avail = bpf_ringbuf_query(&events, BPF_RB_AVAIL_DATA);
-    return avail >= WAKEUP_DATA_SIZE ? BPF_RB_FORCE_WAKEUP : BPF_RB_NO_WAKEUP;
+/* Bytes submitted per CPU since the last forced wakeup.
+ *
+ * Counting what this CPU produced, rather than asking the ring buffer how much
+ * is unconsumed, keeps the decision on the producer side: bpf_ringbuf_query()
+ * reads the consumer position, a cache line the polling thread on another CPU
+ * writes continuously, so querying it per event costs a cross-CPU miss on every
+ * event. */
+BPF_PERCPU_ARRAY_MAP(submitted_bytes, __u64, 1);
+
+static __always_inline long wake_flags(__u64 event_size) {
+    __u32 zero = 0;
+    __u64* pending = bpf_map_lookup_elem(&submitted_bytes, &zero);
+    if (!pending) {
+        /* Can't track the watermark, so don't risk a stalled consumer. */
+        return BPF_RB_FORCE_WAKEUP;
+    }
+
+    *pending += event_size;
+    if (*pending < WAKEUP_DATA_SIZE) {
+        return BPF_RB_NO_WAKEUP;
+    }
+
+    *pending = 0;
+    return BPF_RB_FORCE_WAKEUP;
 }
 
 /* Per-thread scratch for the allocator entry/exit hand-off.
@@ -144,7 +164,7 @@ static __always_inline struct memtrack_task_state* take_slot(enum arg_slot slot)
                                                                         \
         fill_data;                                                      \
                                                                         \
-        bpf_ringbuf_submit(e, wake_flags());                  \
+        bpf_ringbuf_submit(e, wake_flags(sizeof(*e)));                  \
         return 0;                                                       \
     }
 

crates/memtrack/src/ebpf/c/utils/map_helpers.h

@@ -28,6 +28,14 @@
         __type(value, value_type);               \
     } name SEC(".maps")
 
+#define BPF_PERCPU_ARRAY_MAP(name, value_type, max_ents) \
+    struct {                                             \
+        __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);         \
+        __uint(max_entries, max_ents);                   \
+        __type(key, __u32);                              \
+        __type(value, value_type);                       \
+    } name SEC(".maps")
+
 #define BPF_RINGBUF(name, size)             \
     struct {                                \
         __uint(type, BPF_MAP_TYPE_RINGBUF); \