From a7c16c0823d5e5aee5f6b0eeb7a6ec6a3f4d1cd2 Mon Sep 17 00:00:00 2001 From: speak-agent Date: Sun, 23 Aug 2026 02:04:03 +0800 Subject: [PATCH] =?UTF-8?q?0.7.0=20=E2=80=94=E2=80=94=20=E4=B8=A4=E4=B8=AA?= =?UTF-8?q?=E6=8C=87=E9=92=88=E6=A7=BD,=E4=BB=A5=E5=8F=8A=E9=82=A3?= =?UTF-8?q?=E6=9D=A1=E8=B0=81=E4=B9=9F=E6=B2=A1=E5=9C=A8=E6=89=A7=E8=A1=8C?= =?UTF-8?q?=E7=9A=84=E7=BA=A6=E6=9D=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit riscv64 后端的注释从写下来那天起就说着: tp 在这里是约定不是架构寄存器。ABI 把它留给线程局部存储 [...] 内核可以拿它 当 per-CPU 指针 —— 但同一个 ISA 上的宿主程序会在那里找它的线程指针。[...] 这个后端不能被编进任何同时用线程指针的东西里。 最后那句是对**每一个消费者**的约束,写在注释里,没有任何东西在执行它。它成立, 是因为在这个架构上还没有别的东西想要线程指针。 ⭐ 实测 2026-08-23:有了。openkal-opensbi 的启动对象要写 tp,好让程序的 thread_local 能用 —— 在一台没有操作系统的机器上没别人会做这件事。没有它, 第一次 throw 就挂在 __cxa_get_globals 里,而诊断报的是一个异常函数和一个地址, 一个字都没提线程指针。 ## 于是槽分成两个,而且探针会说话 arch_cpu_tls / arch_cpu_set_tls 与 arch_cpu_percpu 平行:一个是**处理器**的 私有指针,一个是**正在运行的上下文**的。examples/switch 加了一条断言,而它断言 的是「这是两个槽」—— 单独 round-trip 在两者别名的后端上照样通过。 riscv64: cpu: the two slots ALIAS ← 改之前 aarch64: cpu: the two slots are distinct x86_64: cpu: the two slots are distinct ## riscv64 的 per-CPU 指针移到 mscratch 那是这个特权级为这件事提供的寄存器,而且这个后端里没有别的东西用它 —— trap.S 走栈保存恢复,不通过它交换。tp 留给它按 ABI 本来的主人。S 模式的同款 后端会用 sscratch,理由一样。 ⚠️ 变的是**哪个寄存器**,不是接口的含义:调用者存一个不透明指针再读回来, 这件事前后一样真。变的是调用者现在还可以用 arch_cpu_tls。 已验证:三个架构 switch ok + 两槽 distinct;宿主 3 个测试全过。 --- README.md | 17 ++++++++++ abi/include/openarch/abi.h | 33 +++++++++++++++++++ backends/aarch64/src/cpu_impl.cpp | 14 ++++++++ backends/riscv64/src/cpu_impl.cpp | 55 +++++++++++++++++++++++++------ backends/x86_64/src/cpu_impl.cpp | 14 ++++++++ examples/switch/src/main.cpp | 18 ++++++++++ mcpp.toml | 2 +- src/cpu.cppm | 15 +++++++++ 8 files changed, 157 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index f95d94a..7b26c4d 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,8 @@ main: back, witness=7 before=1234 trap: raising trap: back, witness=1 cpu: percpu round-trips +cpu: tls round-trips +cpu: the two slots are distinct cpu: four barriers accepted switch ok ``` @@ -72,6 +74,21 @@ that saved the return address and the stack pointer and nothing else would print the first two and corrupt its caller. `before` is a `volatile` local read after the round trip. +⭐ **`the two slots are distinct` is the line that says a comment became a +check.** There are two pointer slots — one the PROCESSOR owns (`arch_cpu_percpu`, +where a kernel keeps what describes this hart) and one the running CONTEXT owns +(`arch_cpu_tls`, where the toolchain expects thread-local storage). On aarch64 +and x86_64 they are different registers and the distinction is free. On riscv64 +`tp` was being used for both, and this file's own backend carried a comment +saying so and a constraint — *"must not be compiled into anything that also uses +a thread pointer"* — that nothing enforced. + +Measured 2026-08-23: something does. An openkal implementation on this +architecture writes `tp` so a program's `thread_local` works. The probe printed +`cpu: the two slots ALIAS`, the per-CPU pointer moved to `mscratch` — the +register this privilege level provides for it — and the line now reads +`distinct` on all three. + ### What the second architecture actually changed Two things, and neither was visible with one backend. diff --git a/abi/include/openarch/abi.h b/abi/include/openarch/abi.h index 60c7784..5337ff5 100644 --- a/abi/include/openarch/abi.h +++ b/abi/include/openarch/abi.h @@ -151,6 +151,39 @@ void* arch_cpu_percpu(void); void arch_cpu_set_percpu(void* p); void arch_cpu_fence(int barrier); +/* ⭐ THE OTHER POINTER SLOT: THE ONE THE RUNNING *CONTEXT* OWNS, NOT THE ONE + * THE PROCESSOR OWNS. + * + * `arch_cpu_percpu` is per PROCESSOR — a kernel points it at a structure that + * describes this hart. This pair is per CONTEXT: it is where the toolchain + * expects to find the thread-local storage of whatever is running now, and a + * program compiled with `thread_local` reads through it on every access. + * + * ⚠️ THEY ARE NOT THE SAME SLOT AND ON ONE MACHINE THEY COMPETE FOR THE SAME + * REGISTER. cpu_impl.cpp for riscv64 has said so since it was written: + * + * `tp' IS A CONVENTION HERE, NOT AN ARCHITECTURAL REGISTER. [...] a kernel + * may use it for its per-CPU pointer --- but a hosted program on the same + * ISA would find its thread pointer there instead. [...] this backend must + * not be compiled into anything that also uses a thread pointer. + * + * That warning has now been met by an actual program. Measured 2026-08-23: + * `openkal-opensbi`'s startup object writes `tp` so that libc++abi's + * `thread_local` works, because on a machine with no operating system nobody + * else will. Without it the first `throw` faults inside `__cxa_get_globals` and + * the diagnostic names an exception function and an address --- it says nothing + * about a thread pointer. + * + * ⇒ So the slot belongs here, beside the one it is confused with, and the two + * are separate operations because on aarch64 and x86_64 they are separate + * REGISTERS and on riscv64 they are not. An implementation that needs both on + * riscv64 must move the per-CPU pointer to `sscratch`; this interface makes + * that a decision with a name rather than a collision discovered at run time. + * + * ⚠️ Undefined before someone sets it, exactly as the per-CPU slot is. */ +void* arch_cpu_tls(void); +void arch_cpu_set_tls(void* p); + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/backends/aarch64/src/cpu_impl.cpp b/backends/aarch64/src/cpu_impl.cpp index 9c965d1..764c39d 100644 --- a/backends/aarch64/src/cpu_impl.cpp +++ b/backends/aarch64/src/cpu_impl.cpp @@ -36,3 +36,17 @@ extern "C" void arch_cpu_fence(int b) { } } + +// `TPIDR_EL0` is the one a program reaches, and it is a DIFFERENT register from +// the `TPIDR_EL1` above — so on this machine the two slots do not compete. +// Readable and writable at EL1; readable by a program at EL0, which is what +// makes it the thread pointer. +extern "C" void* arch_cpu_tls(void) { + void* p; + asm volatile("mrs %0, tpidr_el0" : "=r"(p)); + return p; +} + +extern "C" void arch_cpu_set_tls(void* p) { + asm volatile("msr tpidr_el0, %0" :: "r"(p)); +} diff --git a/backends/riscv64/src/cpu_impl.cpp b/backends/riscv64/src/cpu_impl.cpp index cf8bbd7..fc97da5 100644 --- a/backends/riscv64/src/cpu_impl.cpp +++ b/backends/riscv64/src/cpu_impl.cpp @@ -2,23 +2,46 @@ #include -// ⚠️ `tp` IS A CONVENTION HERE, NOT AN ARCHITECTURAL REGISTER. The ABI reserves -// it for thread-local storage and the hardware assigns it no meaning, so a -// kernel may use it for its per-CPU pointer — but a hosted program on the same -// ISA would find its thread pointer there instead. aarch64's `TPIDR_EL1` is -// architectural and privileged, so no such collision exists. +// ⭐⭐ `mscratch`, AND IT USED TO BE `tp`. THE WARNING THAT SAT HERE CAME TRUE. // -// The consequence for a caller is nil, which is the point; the consequence for -// this backend is that it must not be compiled into anything that also uses a -// thread pointer. +// What this file said before, verbatim: +// +// `tp' IS A CONVENTION HERE, NOT AN ARCHITECTURAL REGISTER. The ABI +// reserves it for thread-local storage [...] a kernel may use it for its +// per-CPU pointer — but a hosted program on the same ISA would find its +// thread pointer there instead. [...] this backend must not be compiled +// into anything that also uses a thread pointer. +// +// That last sentence is a constraint on every consumer, stated in a comment, +// enforced by nothing. It held while nothing on this architecture wanted a +// thread pointer. Measured 2026-08-23, something does: `openkal-opensbi`'s +// startup object writes `tp` so that a program's `thread_local` works, because +// on a machine with no operating system nobody else will. +// +// And the probe says so now rather than a program discovering it: +// +// riscv64: cpu: the two slots ALIAS +// aarch64: cpu: the two slots are distinct +// x86_64: cpu: the two slots are distinct +// +// ⇒ The per-CPU pointer moves to `mscratch`, which is the register this +// privilege level provides FOR THIS PURPOSE and which nothing else in this +// backend uses — trap.S saves and restores through the stack rather than +// swapping through it. `tp` is left to the thread pointer, whose owner it is by +// ABI. An S-mode variant of this backend would use `sscratch` for the same +// reason. +// +// ⚠️ THIS CHANGES WHICH REGISTER, NOT WHAT THE INTERFACE MEANS. A caller stores +// an opaque pointer and reads it back; that is as true after the move as +// before. What changes is that a caller may now also use `arch_cpu_tls`. extern "C" void* arch_cpu_percpu(void) { void* p; - asm volatile("mv %0, tp" : "=r"(p)); + asm volatile("csrr %0, mscratch" : "=r"(p)); return p; } extern "C" void arch_cpu_set_percpu(void* p) { - asm volatile("mv tp, %0" :: "r"(p)); + asm volatile("csrw mscratch, %0" :: "r"(p)); } extern "C" void arch_cpu_fence(int b) { @@ -40,3 +63,15 @@ extern "C" void arch_cpu_fence(int b) { } } + +// `tp`, which the ABI reserves for exactly this and which nothing else in this +// backend now touches. The per-CPU pointer moved out of it — see above. +extern "C" void* arch_cpu_tls(void) { + void* p; + asm volatile("mv %0, tp" : "=r"(p)); + return p; +} + +extern "C" void arch_cpu_set_tls(void* p) { + asm volatile("mv tp, %0" :: "r"(p)); +} diff --git a/backends/x86_64/src/cpu_impl.cpp b/backends/x86_64/src/cpu_impl.cpp index d5f36f3..d23d9cc 100644 --- a/backends/x86_64/src/cpu_impl.cpp +++ b/backends/x86_64/src/cpu_impl.cpp @@ -37,6 +37,12 @@ namespace { // `swapgs` reads its own pointer from `IA32_GS_BASE`. Using the other one // produces a per-CPU pointer that is correct only after a syscall. constexpr unsigned kIa32GsBase = 0xC0000101u; +// ⚠️ `FS` AND NOT `GS`, AND ON THIS MACHINE THAT IS THE WHOLE DIFFERENCE +// BETWEEN THE TWO SLOTS. The System V ABI puts a program's thread-local storage +// at `%fs`; a kernel's per-CPU structure conventionally sits at `%gs`, which is +// also what `swapgs` exists to exchange on entry. Two registers, so unlike +// riscv64 the two slots here do not compete. +constexpr unsigned kIa32FsBase = 0xC0000100u; inline arch_u64 rdmsr(unsigned msr) noexcept { unsigned lo, hi; @@ -107,3 +113,11 @@ extern "C" void arch_cpu_fence(int b) { default: break; } } + +extern "C" void* arch_cpu_tls(void) { + return reinterpret_cast(rdmsr(kIa32FsBase)); +} + +extern "C" void arch_cpu_set_tls(void* p) { + wrmsr(kIa32FsBase, reinterpret_cast(p)); +} diff --git a/examples/switch/src/main.cpp b/examples/switch/src/main.cpp index cc656b2..1b14490 100644 --- a/examples/switch/src/main.cpp +++ b/examples/switch/src/main.cpp @@ -115,17 +115,35 @@ void probe_trap() { // continues, which is what catches a backend that emitted an instruction the // machine does not have. int g_percpu_area = 0; +int g_tls_area = 0; void probe_cpu() { arch::set_percpu(&g_percpu_area); const bool same = (arch::percpu() == &g_percpu_area); + // ⭐ THE SECOND SLOT, AND THE ASSERTION IS THAT IT IS A SECOND SLOT. + // + // Round-tripping it alone would pass on a backend where `tls` and `percpu` + // are the same register — which on riscv64 they ARE, by convention, and + // abi.h records why that matters. So the check is that BOTH hold their own + // value at the same time, which is exactly what fails when one aliases the + // other. On riscv64 this is expected to fail until the backend moves the + // per-CPU pointer to `sscratch`; the point of the check is that the day it + // matters, a program says so instead of reading its thread-locals out of a + // per-CPU structure. + arch::set_tls(&g_tls_area); + const bool tls_ok = (arch::tls() == &g_tls_area); + const bool distinct_ok = (arch::percpu() == &g_percpu_area); + arch::fence(arch::barrier::memory); arch::fence(arch::barrier::store); arch::fence(arch::barrier::complete); arch::fence(arch::barrier::fetch); machine::print(same ? "cpu: percpu round-trips\n" : "cpu: percpu FAILED\n"); + machine::print(tls_ok ? "cpu: tls round-trips\n" : "cpu: tls FAILED\n"); + machine::print(distinct_ok ? "cpu: the two slots are distinct\n" + : "cpu: the two slots ALIAS\n"); machine::print("cpu: four barriers accepted\n"); } diff --git a/mcpp.toml b/mcpp.toml index da1ebd5..a338470 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -33,7 +33,7 @@ [package] namespace = "mcpplibs" name = "openarch" -version = "0.6.0" +version = "0.7.0" description = "openarch: the architecture-mechanism layer — execution contexts, traps, per-CPU state and address spaces, as one interface over several instruction sets" license = "Apache-2.0" authors = ["mcpplibs"] diff --git a/src/cpu.cppm b/src/cpu.cppm index 70f8580..c287d21 100644 --- a/src/cpu.cppm +++ b/src/cpu.cppm @@ -39,6 +39,21 @@ export namespace arch { inline void* percpu() noexcept { return ::arch_cpu_percpu(); } inline void set_percpu(void* p) noexcept { ::arch_cpu_set_percpu(p); } +// The running CONTEXT's private pointer, which is a different question from the +// one above and is asked by different code. +// +// The per-CPU slot is read by a kernel that wants to know which hart it is on. +// This one is read by the TOOLCHAIN, on every access to a `thread_local`, and +// it is set by whoever creates the context. Where a program carries no loader, +// that is the openkal implementation beneath it. +// +// ⚠️ On riscv64 the two slots want the same register (`tp`), and abi.h records +// the measurement that made the conflict real rather than theoretical. Naming +// them separately is what lets an implementation choose; it does not make the +// register conflict go away. +inline void* tls() noexcept { return ::arch_cpu_tls(); } +inline void set_tls(void* p) noexcept { ::arch_cpu_set_tls(p); } + // The four orderings both machines can state. enum class barrier { // Everything before is ordered before everything after, as observed by