From 479d5e5d07b4304bec6dfa37533d45086c4e5194 Mon Sep 17 00:00:00 2001 From: Nicolas Maman Date: Fri, 18 Sep 2026 00:28:28 -0300 Subject: [PATCH] gen-orchestrator: emit the per-node bookkeeping once, not per node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fan-out of ~140 nodes could not build: the generated orchestrator carried ~360 tokens of inline bookkeeping per node (selector gate, rc capture, status, telemetry record), so aether-ui's 138-node tree sat at 49,800 tokens and one more .build.ae tripped aetherc's 50,000-token cap on a main file — "source file exceeds maximum token limit" with no file named (aether-ui #140). The bookkeeping is now two helpers emitted once (_aeb_selected, _aeb_record); each node is a guarded extern call plus one record call, ~35 tokens. Same 138 nodes: 49,770 → 7,227 tokens. Semantics unchanged. tests/test_gen_orchestrator_budget.ae generates a 500-node orchestrator (~180k tokens under the old scheme) and requires `ae check` to accept it against lib/bldr; it fails 4/4 against the previous generator. --- CHANGELOG.md | 16 +++ tests/test_gen_orchestrator_budget.ae | 115 +++++++++++++++++++++ tools/gen-orchestrator.ae | 143 ++++++++++++++------------ 3 files changed, 211 insertions(+), 63 deletions(-) create mode 100644 tests/test_gen_orchestrator_budget.ae diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bfacd37..aff13992 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,22 @@ ### Fixed +- **A fan-out with ~140 nodes could not build at all: the generated + orchestrator blew aetherc's 50,000-token cap on a main file.** + `gen-orchestrator` emitted the whole per-node bookkeeping (selector gate, + rc capture, status, the telemetry record) INLINE for every node — ~360 + tokens each — so aether-ui's 138-node tree sat at 49,800 tokens and the + next `.build.ae` anyone added failed with `source file exceeds maximum + token limit (50000 tokens)` with no file named and nothing the author + touched implicated (aether-ui #140). The bookkeeping is now two helpers + emitted once (`_aeb_selected`, `_aeb_record`) and a node costs ~35 tokens: + the same 138-node orchestrator is 7,227 tokens, headroom for well over a + thousand nodes. Behaviour is unchanged — same selector semantics, same + rc→`_mark_failed` marking, same record fields. Pinned by + `tests/test_gen_orchestrator_budget.ae`, which generates a 500-node + orchestrator (~180,000 tokens under the old scheme) and requires + `ae check` to accept it against the real `lib/bldr`. + - **`make install` destroyed `tools/aeb-resolve.jar`, breaking every maven/java/scala/kotlin build.** The install does `rm -rf $(SHAREDIR)/tools` then `cp -R tools $(SHAREDIR)/tools`, but the diff --git a/tests/test_gen_orchestrator_budget.ae b/tests/test_gen_orchestrator_budget.ae new file mode 100644 index 00000000..2c24f5e0 --- /dev/null +++ b/tests/test_gen_orchestrator_budget.ae @@ -0,0 +1,115 @@ +// gen-orchestrator's output must stay inside aetherc's per-file token cap +// however many nodes the fan-out has. +// +// aetherc lexes a MAIN file into a fixed 50,000-token array (modules get +// 100,000). The orchestrator used to emit ~360 tokens of bookkeeping per +// node inline, so a 138-node fan-out (aether-ui) sat at 49,800 tokens and +// the next `.build.ae` anyone added failed the whole build with "source +// file exceeds maximum token limit" — no file named, nothing the author +// touched. The bookkeeping now lives in two helpers emitted once, and a +// node costs ~35 tokens. +// +// This pins that: generate an orchestrator for 500 synthetic nodes (which +// would have been ~180,000 tokens under the old scheme) and require that +// `ae check` accepts it against the real lib/bldr. A regression back to +// inline emission cannot pass the cap at that count. +// +// gen-orchestrator is a compiled tool, not a lib, so it runs as a +// subprocess — built on demand with the harness's compiler ($AETHER, +// exported by tests/run.sh), the same way test_extract_deps_scan does. + +import std.spec +import std.string +import std.os +import std.file +import std.fs +import std.io +import std.path + +import bldr (_sh, _sh_capture) + +_mktempdir() { + // Native form on Windows (see test_extract_deps_scan._mktempdir). + raw, _err = _sh_capture("mktemp -d") + d = string.trim(raw) + nat, _nerr = _sh_capture("cygpath -m ${d}") + n = string.trim(nat) + if string.length(n) > 0 { return n } + return d +} + +_ae_bin() { + ae_bin = os.getenv("AETHER") + if string.length(ae_bin) == 0 { ae_bin = "ae" } + return ae_bin +} + +_ensure_gen_orchestrator_built() { + if file.exists("tools/gen-orchestrator") == 1 { return 0 } + if file.exists("tools/gen-orchestrator.exe") == 1 { return 0 } + _sh("${_ae_bin()} build tools/gen-orchestrator.ae -o tools/gen-orchestrator --lib lib --lib tools >/dev/null 2>&1") + return 0 +} + +// The synthetic node list: dir_/.build.ae, one per line, as a shell +// script that expands them onto gen-orchestrator's argv. +_write_node_script(dir: string, n: int) { + abs_raw, _e = _sh_capture("readlink -f tools/gen-orchestrator") + gen = string.trim(abs_raw) + body = "#!/bin/sh\nset -e\nargs=\"\"\ni=0\nwhile [ $i -lt ${n} ]; do\n args=\"$args node_$i/.build.ae\"\n i=$((i+1))\ndone\nexec '${gen}' $args\n" + fs.write_atomic(path.join(dir, "gen.sh"), body, string.length(body)) + return 0 +} + +main() { + fw = spec.init() + + spec.describe(fw, "gen-orchestrator stays inside the main-file token cap") { + spec.it("a 500-node orchestrator type-checks against lib/bldr") callback { + _ensure_gen_orchestrator_built() + dir = _mktempdir() + _write_node_script(dir, 500) + out = path.join(dir, "_orchestrator.ae") + _sh("sh '${dir}/gen.sh' > '${out}'") + + spec.assert_true(file.exists(out), "gen-orchestrator wrote the orchestrator") + + // One helper call per node, not a copy of the bookkeeping. + src = io_read(out) + spec.assert_eq(_count(src, "_aeb_record(s, _root, _records, "), 500, + "every node records through the shared helper") + spec.assert_eq(_count(src, "_aeb_record(s: ptr"), 1, + "the record helper is emitted exactly once") + spec.assert_eq(_count(src, "_aeb_selected(sel: string"), 1, + "the selector helper is emitted exactly once") + + // The real gate: aetherc's lexer runs before the type-checker, so + // an over-budget file fails here with "exceeds maximum token + // limit" whatever else is right about it. + rc = _sh("${_ae_bin()} check --lib lib '${out}' >/dev/null 2>&1") + spec.assert_eq(rc, 0, "ae check accepts a 500-node orchestrator (under the 50,000-token cap)") + + _sh("rm -rf '${dir}'") + } + } + + return spec.run_summary(fw) +} + +io_read(p: string) -> string { + s, _err = io.read_file(p) + return s +} + +_count(hay: string, needle: string) -> int { + n = 0 + at = 0 + nl = string.length(needle) + while 1 == 1 { + i = string.index_of_from(hay, needle, at) + if i < 0 { return n } + n = n + 1 + at = i + nl + } + return n +} diff --git a/tools/gen-orchestrator.ae b/tools/gen-orchestrator.ae index be8df98c..a9a20e8e 100644 --- a/tools/gen-orchestrator.ae +++ b/tools/gen-orchestrator.ae @@ -71,6 +71,72 @@ main() { } println("") + // The per-node bookkeeping (selector gate, rc capture, status, the + // telemetry record) used to be emitted INLINE for every node: ~360 + // tokens each, so a 138-node fan-out (aether-ui) produced a 49,800-token + // orchestrator and aetherc's 50,000-token cap on a main file failed the + // whole build the day one more `.build.ae` was added — "source file + // exceeds maximum token limit" with no file named, pointing at nothing + // the author touched. It is emitted ONCE here as two helpers, and each + // node costs ~35 tokens: the only thing that genuinely differs per node + // is the extern it calls, which cannot be a value (a fn passed from a + // separately-compiled TU is DCE'd, aether #2037), so that one call stays + // inline and everything around it moves here. + // + // Same TU as main(), so bldr's cloned-from-import statics are visible. + println("_aeb_selected(sel: string, label: string) -> int {") + println(" if string.length(sel) == 0 { return 1 }") + println(" if string.equals(sel, label) == 1 { return 1 }") + println(" return 0") + println("}") + println("") + // A non-zero node return that did NOT already go through bldr.fail() + // (e.g. a custom gate that just `return 1`s) is marked failed here, so + // both the status/JSON and the any_failed gate in main() see it. The + // explicit bldr.fail() channel still works and takes precedence (a node + // that failed via fail() stays failed regardless of its rc). Marked at + // the SESSION level: bldr.fail() takes a NODE ctx, we hold the session. + println("_aeb_record(s: ptr, root: string, records: ptr, label: string, type_word: string, nrc: int, start: long) {") + println(" if nrc != 0 { bldr._mark_failed(s, label, string.concat(\"node returned non-zero rc=\", string.from_int(nrc))) }") + println(" bldr.done(s, label)") + println(" end = clock_ns()") + println(" rec = map.new()") + println(" map.put(rec, \"label\", label)") + println(" map.put(rec, \"type\", type_word)") + println(" map.put(rec, \"wall_ms\", string.from_int((end - start) / 1000000))") + println(" st = bldr.status_of(s, label)") + println(" if string.length(st) == 0 { st = \"passed\" }") + println(" map.put(rec, \"status\", st)") + // rc reflects the node's logical outcome so the JSON's rc agrees with + // status (a failed test can't report rc:0). The in-process orchestrator + // has no child exit code to read — status IS the outcome here — so map + // failed→1, everything else→0. (The driver's per-node make path reads a + // real child rc from the .rc marker.) + println(" rc = \"0\"") + println(" if string.equals(st, \"failed\") == 1 { rc = \"1\" }") + println(" map.put(rec, \"rc\", rc)") + println(" td = bldr._label_to_target_dir(root, label)") + println(" map.put(rec, \"cache\", bldr._read_cache_outcome(td))") + // Read the test-result marker for EVERY node — gated at runtime on the + // marker existing (thr), NOT on the type name. A node whose build wrote + // pass/fail markers (via a *_test SDK verb) gets the row whatever its + // filename; one that didn't leaves thr=0 and renders the plain line. + // No "type == test" inference. + println(" tp, tf, tsk, thr = bldr._read_test_result(td)") + println(" if thr == 1 {") + println(" map.put(rec, \"test_passed\", string.from_int(tp))") + println(" map.put(rec, \"test_failed\", string.from_int(tf))") + println(" map.put(rec, \"test_skipped\", string.from_int(tsk))") + println(" map.put(rec, \"test_has_report\", string.from_int(bldr._read_test_report_flag(td)))") + println(" }") + println(" tfn = bldr._read_test_failures(td)") + println(" if string.length(tfn) > 0 {") + println(" map.put(rec, \"test_failed_names\", tfn)") + println(" }") + println(" list.add(records, rec)") + println("}") + println("") + // main() calls everything in topo order, accumulating per-module // telemetry records (label, type, wall_ms, cache outcome). At the // end it hands the records list to bldr.render_telemetry for the @@ -102,69 +168,20 @@ main() { // .install.ae→install, .essais.ae→essais. (The route IS the filename.) type = infer_type(file) type_word = type - // Per-node selector gate (slice A): run this node iff no - // selector was given (all-in-one) or it names this label. - println(" _run = 0") - println(" if string.length(_sel) == 0 { _run = 1 }") - println(" if string.equals(_sel, \"${label}\") == 1 { _run = 1 }") - println(" if _run == 1 {") - println(" _start = clock_ns()") - // Run the node and CAPTURE its return. This is now trustworthy: every - // SDK builder is declared `: int` (whole-tree sweep) and transform-ae - // rewrites the node entry to `(s: ptr): int {`, so a node whose - // last statement is a bare builder call flows that builder's real rc as - // its own return — no longer the void-through-`extern -> int` register - // garbage that forced us to discard it (which in turn let a hand-written - // node `return 1` exit falsely-green — the presubmit bug, - // asks/node-nonzero-return-not-propagated-to-exit-code.md). - // - // A non-zero node return that did NOT already go through bldr.fail() - // (e.g. a custom gate that just `return 1`s) is marked failed here, so - // both the status/JSON and the any_failed gate below see it. The - // explicit bldr.fail() channel still works and takes precedence (a - // node that failed via fail() stays failed regardless of its rc). - println(" _nrc = ${fname}(s)") - // Mark failed at the SESSION level: bldr.fail() takes a NODE ctx (it - // reads _session/module_dir off it) — we hold the session s here, so - // call the session primitive bldr._mark_failed(s, label, reason) - // directly. This is the same list any_failed(s) reads below. - println(" if _nrc != 0 { bldr._mark_failed(s, \"${label}\", string.concat(\"node returned non-zero rc=\", string.from_int(_nrc))) }") - println(" bldr.done(s, \"${label}\")") - println(" _end = clock_ns()") - println(" _rec = map.new()") - println(" map.put(_rec, \"label\", \"${label}\")") - println(" map.put(_rec, \"type\", \"${type_word}\")") - println(" map.put(_rec, \"wall_ms\", string.from_int((_end - _start) / 1000000))") - println(" _st = bldr.status_of(s, \"${label}\")") - println(" if string.length(_st) == 0 { _st = \"passed\" }") - println(" map.put(_rec, \"status\", _st)") - // rc reflects the node's logical outcome so the JSON's rc agrees - // with status (a failed test can't report rc:0). The in-process - // orchestrator has no child exit code to read — status IS the - // outcome here — so map failed→1, everything else→0. (The driver's - // per-node make path reads a real child rc from the .rc marker.) - println(" _rc = \"0\"") - println(" if string.equals(_st, \"failed\") == 1 { _rc = \"1\" }") - println(" map.put(_rec, \"rc\", _rc)") - println(" _td = bldr._label_to_target_dir(_root, \"${label}\")") - println(" map.put(_rec, \"cache\", bldr._read_cache_outcome(_td))") - // Read the test-result marker for EVERY node — gated at runtime on the - // marker existing (_thr), NOT on the type name. A node whose build wrote - // pass/fail markers (via a *_test SDK verb) gets the row whatever its - // filename; one that didn't leaves _thr=0 and renders the plain line. - // No "type == test" inference. - println(" _tp, _tf, _tsk, _thr = bldr._read_test_result(_td)") - println(" if _thr == 1 {") - println(" map.put(_rec, \"test_passed\", string.from_int(_tp))") - println(" map.put(_rec, \"test_failed\", string.from_int(_tf))") - println(" map.put(_rec, \"test_skipped\", string.from_int(_tsk))") - println(" map.put(_rec, \"test_has_report\", string.from_int(bldr._read_test_report_flag(_td)))") - println(" }") - println(" _tfn = bldr._read_test_failures(_td)") - println(" if string.length(_tfn) > 0 {") - println(" map.put(_rec, \"test_failed_names\", _tfn)") - println(" }") - println(" list.add(_records, _rec)") + // Per-node selector gate (slice A): run this node iff no selector + // was given (all-in-one) or it names this label. Then run the node + // and CAPTURE its return. This is trustworthy: every SDK builder is + // declared `: int` (whole-tree sweep) and transform-ae rewrites the + // node entry to `(s: ptr): int {`, so a node whose last + // statement is a bare builder call flows that builder's real rc as + // its own return — no longer the void-through-`extern -> int` + // register garbage that forced us to discard it (which in turn let a + // hand-written node `return 1` exit falsely-green — the presubmit + // bug, asks/node-nonzero-return-not-propagated-to-exit-code.md). + println(" if _aeb_selected(_sel, \"${label}\") == 1 {") + println(" _start = clock_ns()") + println(" _nrc = ${fname}(s)") + println(" _aeb_record(s, _root, _records, \"${label}\", \"${type_word}\", _nrc, _start)") println(" }") i = i + 1 }