Skip to content

gen-orchestrator: emit the per-node bookkeeping once, not per node #15

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nicolas-maman merged 1 commit into from
Sep 18, 2026
Merged

gen-orchestrator: emit the per-node bookkeeping once, not per node #15

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,22 @@

### Fixed

- **A fan-out with ~140 nodes could not build at all: the generated
orchestrator blew aetherc's 50,000-token cap on a main file.**
`gen-orchestrator` emitted the whole per-node bookkeeping (selector gate,
rc capture, status, the telemetry record) INLINE for every node — ~360
tokens each — so aether-ui's 138-node tree sat at 49,800 tokens and the
next `.build.ae` anyone added failed with `source file exceeds maximum
token limit (50000 tokens)` with no file named and nothing the author
touched implicated (aether-ui #140). The bookkeeping is now two helpers
emitted once (`_aeb_selected`, `_aeb_record`) and a node costs ~35 tokens:
the same 138-node orchestrator is 7,227 tokens, headroom for well over a
thousand nodes. Behaviour is unchanged — same selector semantics, same
rc→`_mark_failed` marking, same record fields. Pinned by
`tests/test_gen_orchestrator_budget.ae`, which generates a 500-node
orchestrator (~180,000 tokens under the old scheme) and requires
`ae check` to accept it against the real `lib/bldr`.

- **`make install` destroyed `tools/aeb-resolve.jar`, breaking every
maven/java/scala/kotlin build.** The install does
`rm -rf $(SHAREDIR)/tools` then `cp -R tools $(SHAREDIR)/tools`, but the
Expand Down
115 changes: 115 additions & 0 deletions tests/test_gen_orchestrator_budget.ae
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
// gen-orchestrator's output must stay inside aetherc's per-file token cap
// however many nodes the fan-out has.
//
// aetherc lexes a MAIN file into a fixed 50,000-token array (modules get
// 100,000). The orchestrator used to emit ~360 tokens of bookkeeping per
// node inline, so a 138-node fan-out (aether-ui) sat at 49,800 tokens and
// the next `.build.ae` anyone added failed the whole build with "source
// file exceeds maximum token limit" — no file named, nothing the author
// touched. The bookkeeping now lives in two helpers emitted once, and a
// node costs ~35 tokens.
//
// This pins that: generate an orchestrator for 500 synthetic nodes (which
// would have been ~180,000 tokens under the old scheme) and require that
// `ae check` accepts it against the real lib/bldr. A regression back to
// inline emission cannot pass the cap at that count.
//
// gen-orchestrator is a compiled tool, not a lib, so it runs as a
// subprocess — built on demand with the harness's compiler ($AETHER,
// exported by tests/run.sh), the same way test_extract_deps_scan does.

import std.spec
import std.string
import std.os
import std.file
import std.fs
import std.io
import std.path

import bldr (_sh, _sh_capture)

_mktempdir() {
// Native form on Windows (see test_extract_deps_scan._mktempdir).
raw, _err = _sh_capture("mktemp -d")
d = string.trim(raw)
nat, _nerr = _sh_capture("cygpath -m ${d}")
n = string.trim(nat)
if string.length(n) > 0 { return n }
return d
}

_ae_bin() {
ae_bin = os.getenv("AETHER")
if string.length(ae_bin) == 0 { ae_bin = "ae" }
return ae_bin
}

_ensure_gen_orchestrator_built() {
if file.exists("tools/gen-orchestrator") == 1 { return 0 }
if file.exists("tools/gen-orchestrator.exe") == 1 { return 0 }
_sh("${_ae_bin()} build tools/gen-orchestrator.ae -o tools/gen-orchestrator --lib lib --lib tools >/dev/null 2>&1")
return 0
}

// The synthetic node list: dir_<i>/.build.ae, one per line, as a shell
// script that expands them onto gen-orchestrator's argv.
_write_node_script(dir: string, n: int) {
abs_raw, _e = _sh_capture("readlink -f tools/gen-orchestrator")
gen = string.trim(abs_raw)
body = "#!/bin/sh\nset -e\nargs=\"\"\ni=0\nwhile [ $i -lt ${n} ]; do\n args=\"$args node_$i/.build.ae\"\n i=$((i+1))\ndone\nexec '${gen}' $args\n"
fs.write_atomic(path.join(dir, "gen.sh"), body, string.length(body))
return 0
}

main() {
fw = spec.init()

spec.describe(fw, "gen-orchestrator stays inside the main-file token cap") {
spec.it("a 500-node orchestrator type-checks against lib/bldr") callback {
_ensure_gen_orchestrator_built()
dir = _mktempdir()
_write_node_script(dir, 500)
out = path.join(dir, "_orchestrator.ae")
_sh("sh '${dir}/gen.sh' > '${out}'")

spec.assert_true(file.exists(out), "gen-orchestrator wrote the orchestrator")

// One helper call per node, not a copy of the bookkeeping.
src = io_read(out)
spec.assert_eq(_count(src, "_aeb_record(s, _root, _records, "), 500,
"every node records through the shared helper")
spec.assert_eq(_count(src, "_aeb_record(s: ptr"), 1,
"the record helper is emitted exactly once")
spec.assert_eq(_count(src, "_aeb_selected(sel: string"), 1,
"the selector helper is emitted exactly once")

// The real gate: aetherc's lexer runs before the type-checker, so
// an over-budget file fails here with "exceeds maximum token
// limit" whatever else is right about it.
rc = _sh("${_ae_bin()} check --lib lib '${out}' >/dev/null 2>&1")
spec.assert_eq(rc, 0, "ae check accepts a 500-node orchestrator (under the 50,000-token cap)")

_sh("rm -rf '${dir}'")
}
}

return spec.run_summary(fw)
}

io_read(p: string) -> string {
s, _err = io.read_file(p)
return s
}

_count(hay: string, needle: string) -> int {
n = 0
at = 0
nl = string.length(needle)
while 1 == 1 {
i = string.index_of_from(hay, needle, at)
if i < 0 { return n }
n = n + 1
at = i + nl
}
return n
}
143 changes: 80 additions & 63 deletions tools/gen-orchestrator.ae
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,72 @@ main() {
}
println("")

// The per-node bookkeeping (selector gate, rc capture, status, the
// telemetry record) used to be emitted INLINE for every node: ~360
// tokens each, so a 138-node fan-out (aether-ui) produced a 49,800-token
// orchestrator and aetherc's 50,000-token cap on a main file failed the
// whole build the day one more `.build.ae` was added — "source file
// exceeds maximum token limit" with no file named, pointing at nothing
// the author touched. It is emitted ONCE here as two helpers, and each
// node costs ~35 tokens: the only thing that genuinely differs per node
// is the extern it calls, which cannot be a value (a fn passed from a
// separately-compiled TU is DCE'd, aether #2037), so that one call stays
// inline and everything around it moves here.
//
// Same TU as main(), so bldr's cloned-from-import statics are visible.
println("_aeb_selected(sel: string, label: string) -> int {")
println(" if string.length(sel) == 0 { return 1 }")
println(" if string.equals(sel, label) == 1 { return 1 }")
println(" return 0")
println("}")
println("")
// A non-zero node return that did NOT already go through bldr.fail()
// (e.g. a custom gate that just `return 1`s) is marked failed here, so
// both the status/JSON and the any_failed gate in main() see it. The
// explicit bldr.fail() channel still works and takes precedence (a node
// that failed via fail() stays failed regardless of its rc). Marked at
// the SESSION level: bldr.fail() takes a NODE ctx, we hold the session.
println("_aeb_record(s: ptr, root: string, records: ptr, label: string, type_word: string, nrc: int, start: long) {")
println(" if nrc != 0 { bldr._mark_failed(s, label, string.concat(\"node returned non-zero rc=\", string.from_int(nrc))) }")
println(" bldr.done(s, label)")
println(" end = clock_ns()")
println(" rec = map.new()")
println(" map.put(rec, \"label\", label)")
println(" map.put(rec, \"type\", type_word)")
println(" map.put(rec, \"wall_ms\", string.from_int((end - start) / 1000000))")
println(" st = bldr.status_of(s, label)")
println(" if string.length(st) == 0 { st = \"passed\" }")
println(" map.put(rec, \"status\", st)")
// rc reflects the node's logical outcome so the JSON's rc agrees with
// status (a failed test can't report rc:0). The in-process orchestrator
// has no child exit code to read — status IS the outcome here — so map
// failed→1, everything else→0. (The driver's per-node make path reads a
// real child rc from the .rc marker.)
println(" rc = \"0\"")
println(" if string.equals(st, \"failed\") == 1 { rc = \"1\" }")
println(" map.put(rec, \"rc\", rc)")
println(" td = bldr._label_to_target_dir(root, label)")
println(" map.put(rec, \"cache\", bldr._read_cache_outcome(td))")
// Read the test-result marker for EVERY node — gated at runtime on the
// marker existing (thr), NOT on the type name. A node whose build wrote
// pass/fail markers (via a *_test SDK verb) gets the row whatever its
// filename; one that didn't leaves thr=0 and renders the plain line.
// No "type == test" inference.
println(" tp, tf, tsk, thr = bldr._read_test_result(td)")
println(" if thr == 1 {")
println(" map.put(rec, \"test_passed\", string.from_int(tp))")
println(" map.put(rec, \"test_failed\", string.from_int(tf))")
println(" map.put(rec, \"test_skipped\", string.from_int(tsk))")
println(" map.put(rec, \"test_has_report\", string.from_int(bldr._read_test_report_flag(td)))")
println(" }")
println(" tfn = bldr._read_test_failures(td)")
println(" if string.length(tfn) > 0 {")
println(" map.put(rec, \"test_failed_names\", tfn)")
println(" }")
println(" list.add(records, rec)")
println("}")
println("")

// main() calls everything in topo order, accumulating per-module
// telemetry records (label, type, wall_ms, cache outcome). At the
// end it hands the records list to bldr.render_telemetry for the
Expand Down Expand Up @@ -102,69 +168,20 @@ main() {
// .install.ae→install, .essais.ae→essais. (The route IS the filename.)
type = infer_type(file)
type_word = type
// Per-node selector gate (slice A): run this node iff no
// selector was given (all-in-one) or it names this label.
println(" _run = 0")
println(" if string.length(_sel) == 0 { _run = 1 }")
println(" if string.equals(_sel, \"${label}\") == 1 { _run = 1 }")
println(" if _run == 1 {")
println(" _start = clock_ns()")
// Run the node and CAPTURE its return. This is now trustworthy: every
// SDK builder is declared `: int` (whole-tree sweep) and transform-ae
// rewrites the node entry to `<fname>(s: ptr): int {`, so a node whose
// last statement is a bare builder call flows that builder's real rc as
// its own return — no longer the void-through-`extern -> int` register
// garbage that forced us to discard it (which in turn let a hand-written
// node `return 1` exit falsely-green — the presubmit bug,
// asks/node-nonzero-return-not-propagated-to-exit-code.md).
//
// A non-zero node return that did NOT already go through bldr.fail()
// (e.g. a custom gate that just `return 1`s) is marked failed here, so
// both the status/JSON and the any_failed gate below see it. The
// explicit bldr.fail() channel still works and takes precedence (a
// node that failed via fail() stays failed regardless of its rc).
println(" _nrc = ${fname}(s)")
// Mark failed at the SESSION level: bldr.fail() takes a NODE ctx (it
// reads _session/module_dir off it) — we hold the session s here, so
// call the session primitive bldr._mark_failed(s, label, reason)
// directly. This is the same list any_failed(s) reads below.
println(" if _nrc != 0 { bldr._mark_failed(s, \"${label}\", string.concat(\"node returned non-zero rc=\", string.from_int(_nrc))) }")
println(" bldr.done(s, \"${label}\")")
println(" _end = clock_ns()")
println(" _rec = map.new()")
println(" map.put(_rec, \"label\", \"${label}\")")
println(" map.put(_rec, \"type\", \"${type_word}\")")
println(" map.put(_rec, \"wall_ms\", string.from_int((_end - _start) / 1000000))")
println(" _st = bldr.status_of(s, \"${label}\")")
println(" if string.length(_st) == 0 { _st = \"passed\" }")
println(" map.put(_rec, \"status\", _st)")
// rc reflects the node's logical outcome so the JSON's rc agrees
// with status (a failed test can't report rc:0). The in-process
// orchestrator has no child exit code to read — status IS the
// outcome here — so map failed→1, everything else→0. (The driver's
// per-node make path reads a real child rc from the .rc marker.)
println(" _rc = \"0\"")
println(" if string.equals(_st, \"failed\") == 1 { _rc = \"1\" }")
println(" map.put(_rec, \"rc\", _rc)")
println(" _td = bldr._label_to_target_dir(_root, \"${label}\")")
println(" map.put(_rec, \"cache\", bldr._read_cache_outcome(_td))")
// Read the test-result marker for EVERY node — gated at runtime on the
// marker existing (_thr), NOT on the type name. A node whose build wrote
// pass/fail markers (via a *_test SDK verb) gets the row whatever its
// filename; one that didn't leaves _thr=0 and renders the plain line.
// No "type == test" inference.
println(" _tp, _tf, _tsk, _thr = bldr._read_test_result(_td)")
println(" if _thr == 1 {")
println(" map.put(_rec, \"test_passed\", string.from_int(_tp))")
println(" map.put(_rec, \"test_failed\", string.from_int(_tf))")
println(" map.put(_rec, \"test_skipped\", string.from_int(_tsk))")
println(" map.put(_rec, \"test_has_report\", string.from_int(bldr._read_test_report_flag(_td)))")
println(" }")
println(" _tfn = bldr._read_test_failures(_td)")
println(" if string.length(_tfn) > 0 {")
println(" map.put(_rec, \"test_failed_names\", _tfn)")
println(" }")
println(" list.add(_records, _rec)")
// Per-node selector gate (slice A): run this node iff no selector
// was given (all-in-one) or it names this label. Then run the node
// and CAPTURE its return. This is trustworthy: every SDK builder is
// declared `: int` (whole-tree sweep) and transform-ae rewrites the
// node entry to `<fname>(s: ptr): int {`, so a node whose last
// statement is a bare builder call flows that builder's real rc as
// its own return — no longer the void-through-`extern -> int`
// register garbage that forced us to discard it (which in turn let a
// hand-written node `return 1` exit falsely-green — the presubmit
// bug, asks/node-nonzero-return-not-propagated-to-exit-code.md).
println(" if _aeb_selected(_sel, \"${label}\") == 1 {")
println(" _start = clock_ns()")
println(" _nrc = ${fname}(s)")
println(" _aeb_record(s, _root, _records, \"${label}\", \"${type_word}\", _nrc, _start)")
println(" }")
i = i + 1
}
Expand Down
Loading