From eead4548f01c4dfd0d3cdeae4e0ffadf8de10770 Mon Sep 17 00:00:00 2001 From: Zhiyu Cheng Date: Thu, 13 Aug 2026 12:20:08 -0700 Subject: [PATCH 1/3] Pin GDPVal evaluator launcher to 0.2.6 Signed-off-by: Zhiyu Cheng --- plugins/modelopt/skills/evaluation/SKILL.md | 4 +- .../gym_gdpval/example_gym_gdpval.yaml | 2 + .../evaluation/recipes/tasks/aa_gym/gdpval.md | 5 +++ .../evaluation/references/gym-gdpval.md | 18 ++++++++- .../skills/evaluation/scripts/nel-gdpval.sh | 37 +++++++++++++++++++ 5 files changed, 64 insertions(+), 2 deletions(-) create mode 100755 plugins/modelopt/skills/evaluation/scripts/nel-gdpval.sh diff --git a/plugins/modelopt/skills/evaluation/SKILL.md b/plugins/modelopt/skills/evaluation/SKILL.md index c0716a71d4d..5df4ce892e3 100644 --- a/plugins/modelopt/skills/evaluation/SKILL.md +++ b/plugins/modelopt/skills/evaluation/SKILL.md @@ -70,7 +70,9 @@ GDPVal: unsandboxed. Verify with `gdpval-sif.sh --check`. `.env` needs `HF_TOKEN`, `INFERENCE_API_KEY`, `TAVILY_API_KEY`, `INFERENCE_JUDGE_URL`, `GDPVAL_SIF_DIR`, and `NEMO_EVALUATOR_TRUST_PRE_CMD=1` (the config has a `pre_cmd`). Thinking mode is mandatory (non-thinking loses ~86%). -4. Dry-run → launch. **`limit_samples` is inert on the gym path** (the gym runs all +4. Run both dry-run and launch through `"$SKILL_DIR/scripts/nel-gdpval.sh"`; it pins + the required 0.2.6 launcher and avoids older launchers' unset + `NEL_INVOCATION_ID` failure. **`limit_samples` is inert on the gym path** (the gym runs all 220 tasks regardless), so there is no cheap canary: watch the real run's first ~20–30 min for the SIF-sandbox line and judge auth, and cancel if wrong. See the recipe's Canary section. diff --git a/plugins/modelopt/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml b/plugins/modelopt/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml index cdc513dc8cd..8d3f1811ca7 100644 --- a/plugins/modelopt/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml +++ b/plugins/modelopt/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml @@ -144,6 +144,8 @@ evaluation: # deliverables are NOT auto-uploaded — they stay on disk for inspection. # Drop "_cache" if you WANT them uploaded as artifacts. PERSIST_DELIVERABLES_DIR: lit:/results/gdpval/deliverables_cache + # Requires launcher 0.2.6 via $SKILL_DIR/scripts/nel-gdpval.sh. That launcher + # assigns this stable ID before the runtime re-export (including across resumes). NEL_INVOCATION_ID: runtime:NEL_INVOCATION_ID # Installs apptainer + squashfuse into the eval container (needs # NEMO_EVALUATOR_TRUST_PRE_CMD=1 in the launching shell). See references/gym-gdpval.md. diff --git a/plugins/modelopt/skills/evaluation/recipes/tasks/aa_gym/gdpval.md b/plugins/modelopt/skills/evaluation/recipes/tasks/aa_gym/gdpval.md index 13a83562f82..a321069469b 100644 --- a/plugins/modelopt/skills/evaluation/recipes/tasks/aa_gym/gdpval.md +++ b/plugins/modelopt/skills/evaluation/recipes/tasks/aa_gym/gdpval.md @@ -15,6 +15,11 @@ in the suite — **220 tasks**, `num_repeats=1`, 4 judge trials per rollout. It runs on the **0.2.6 `nel` launcher** as a `nemo_gym` task (NOT nel-next), so Steps 1–9 apply — but with the branch differences below. +Run it through `"$SKILL_DIR/scripts/nel-gdpval.sh"`, which pins the launcher to +0.2.6. Do not use an unversioned `nel` from PATH: older launchers fail before client +startup when the config forwards `NEL_INVOCATION_ID`. The shared reference explains +the failure signature and dry-run check. + ## What makes GDPVal different (not a normal `aa/` task) - **Standalone** — one gym eval per config. Never add GDPVal to a multi-task diff --git a/plugins/modelopt/skills/evaluation/references/gym-gdpval.md b/plugins/modelopt/skills/evaluation/references/gym-gdpval.md index 61aaf0746f7..8cc7c90c0a0 100644 --- a/plugins/modelopt/skills/evaluation/references/gym-gdpval.md +++ b/plugins/modelopt/skills/evaluation/references/gym-gdpval.md @@ -9,6 +9,22 @@ run **inline in the eval container** (`install_on_the_fly`) via `ng_prepare_benc is `recipes/examples/gym_gdpval/` and the per-task pointer is `recipes/tasks/aa_gym/gdpval.md`. +Always invoke GDPVal through the pinned wrapper, even if `nel` is already on PATH: + +```bash +"$SKILL_DIR/scripts/nel-gdpval.sh" --version # must report nemo_evaluator_launcher: 0.2.6 +"$SKILL_DIR/scripts/nel-gdpval.sh" run --config --dry-run +"$SKILL_DIR/scripts/nel-gdpval.sh" run --config +``` + +This is a correctness requirement, not only a dependency pin. Launcher 0.2.6 writes +the generated `NEL_INVOCATION_ID` into `run.sub` before environment-variable +re-exports. Older launchers can emit `export NEL_INVOCATION_ID="${NEL_INVOCATION_ID}"` +without first assigning it, then exit with `NEL_INVOCATION_ID: unbound variable` +under `set -u` before the evaluation client starts. In a dry-run, verify that a +literal assignment appears before the re-export; do not substitute `SLURM_JOB_ID`, +because it changes across the benchmark's walltime-resume chain. + ## Where each piece runs | Component | Where | @@ -184,7 +200,7 @@ mount source, and `raise ValueError` listing the missing ones **before** any | `GDPVAL_REF_FILES_DIR` | lit:/gdpval_ref_files | shared-FS ref-file staging (node-local /tmp breaks multi-node Ray) | | `PERSIST_DELIVERABLES_DIR` | lit | where deliverables persist (see MLflow note) | | `GDPVAL_MAX_TURNS` | lit (optional) | Stirrup turn cap (default 100; golden uses 250) | -| `NEL_INVOCATION_ID` | runtime | run id | +| `NEL_INVOCATION_ID` | runtime | stable run id assigned by launcher 0.2.6; do not use `SLURM_JOB_ID` | `INFERENCE_JUDGE_URL` is the judge host — config (from `.env`), substituted as the literal `` placeholder in `gdpval_judge.base_url`, **not** diff --git a/plugins/modelopt/skills/evaluation/scripts/nel-gdpval.sh b/plugins/modelopt/skills/evaluation/scripts/nel-gdpval.sh new file mode 100755 index 00000000000..3c91fd9dbd4 --- /dev/null +++ b/plugins/modelopt/skills/evaluation/scripts/nel-gdpval.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# GDPVal requires launcher 0.2.6. In particular, 0.2.6 assigns the generated +# NEL_INVOCATION_ID in run.sub before evaluation.env_vars re-exports it. Older +# launchers can instead fail under `set -u` before the evaluation client starts. +set -euo pipefail + +NEL_GDPVAL_SPEC="${NEL_GDPVAL_SPEC:-nemo-evaluator-launcher[all]==0.2.6}" + +case "${1:-}" in + -h|--help) + echo "usage: nel-gdpval.sh " + echo " nel-gdpval.sh --version" + exit 0 + ;; +esac + +command -v uvx >/dev/null 2>&1 || { + echo "ERROR: 'uvx' is required to run the pinned GDPVal launcher" >&2 + exit 1 +} + +exec uvx --python 3.10 --from "$NEL_GDPVAL_SPEC" nel "$@" From 938abd229745c35f82faf919f0f44c0130738f15 Mon Sep 17 00:00:00 2001 From: Zhiyu Cheng Date: Thu, 13 Aug 2026 16:24:50 -0700 Subject: [PATCH 2/3] Harden GDPVal launcher version pin Signed-off-by: Zhiyu Cheng --- plugins/modelopt/skills/evaluation/SKILL.md | 9 ++-- .../evaluation/recipes/tasks/aa_gym/gdpval.md | 6 +-- .../evaluation/references/gym-gdpval.md | 11 +++-- .../skills/evaluation/scripts/nel-gdpval.sh | 6 +-- .../evaluation/tests/test_nel_gdpval.py | 49 +++++++++++++++++++ 5 files changed, 66 insertions(+), 15 deletions(-) create mode 100644 plugins/modelopt/skills/evaluation/tests/test_nel_gdpval.py diff --git a/plugins/modelopt/skills/evaluation/SKILL.md b/plugins/modelopt/skills/evaluation/SKILL.md index 5df4ce892e3..8940264b819 100644 --- a/plugins/modelopt/skills/evaluation/SKILL.md +++ b/plugins/modelopt/skills/evaluation/SKILL.md @@ -70,10 +70,11 @@ GDPVal: unsandboxed. Verify with `gdpval-sif.sh --check`. `.env` needs `HF_TOKEN`, `INFERENCE_API_KEY`, `TAVILY_API_KEY`, `INFERENCE_JUDGE_URL`, `GDPVAL_SIF_DIR`, and `NEMO_EVALUATOR_TRUST_PRE_CMD=1` (the config has a `pre_cmd`). Thinking mode is mandatory (non-thinking loses ~86%). -4. Run both dry-run and launch through `"$SKILL_DIR/scripts/nel-gdpval.sh"`; it pins - the required 0.2.6 launcher and avoids older launchers' unset - `NEL_INVOCATION_ID` failure. **`limit_samples` is inert on the gym path** (the gym runs all - 220 tasks regardless), so there is no cheap canary: watch the real run's first +4. Run both dry-run and launch through `"$SKILL_DIR/scripts/nel-gdpval.sh"`; it + enforces the required 0.2.6 launcher even if `nel` on PATH is stale and avoids + the observed 0.2.4 launcher's unset `NEL_INVOCATION_ID` failure. + **`limit_samples` is inert on the gym path** (the gym runs all 220 tasks + regardless), so there is no cheap canary: watch the real run's first ~20–30 min for the SIF-sandbox line and judge auth, and cancel if wrong. See the recipe's Canary section. diff --git a/plugins/modelopt/skills/evaluation/recipes/tasks/aa_gym/gdpval.md b/plugins/modelopt/skills/evaluation/recipes/tasks/aa_gym/gdpval.md index a321069469b..e7f838ef679 100644 --- a/plugins/modelopt/skills/evaluation/recipes/tasks/aa_gym/gdpval.md +++ b/plugins/modelopt/skills/evaluation/recipes/tasks/aa_gym/gdpval.md @@ -16,9 +16,9 @@ It runs on the **0.2.6 `nel` launcher** as a `nemo_gym` task (NOT nel-next), so Steps 1–9 apply — but with the branch differences below. Run it through `"$SKILL_DIR/scripts/nel-gdpval.sh"`, which pins the launcher to -0.2.6. Do not use an unversioned `nel` from PATH: older launchers fail before client -startup when the config forwards `NEL_INVOCATION_ID`. The shared reference explains -the failure signature and dry-run check. +0.2.6. Do not use an unversioned `nel` from PATH: the observed 0.2.4 launcher fails +before client startup when the config forwards `NEL_INVOCATION_ID`. The shared +reference explains the failure signature and dry-run check. ## What makes GDPVal different (not a normal `aa/` task) diff --git a/plugins/modelopt/skills/evaluation/references/gym-gdpval.md b/plugins/modelopt/skills/evaluation/references/gym-gdpval.md index 8cc7c90c0a0..6cd3ee279de 100644 --- a/plugins/modelopt/skills/evaluation/references/gym-gdpval.md +++ b/plugins/modelopt/skills/evaluation/references/gym-gdpval.md @@ -19,11 +19,12 @@ Always invoke GDPVal through the pinned wrapper, even if `nel` is already on PAT This is a correctness requirement, not only a dependency pin. Launcher 0.2.6 writes the generated `NEL_INVOCATION_ID` into `run.sub` before environment-variable -re-exports. Older launchers can emit `export NEL_INVOCATION_ID="${NEL_INVOCATION_ID}"` -without first assigning it, then exit with `NEL_INVOCATION_ID: unbound variable` -under `set -u` before the evaluation client starts. In a dry-run, verify that a -literal assignment appears before the re-export; do not substitute `SLURM_JOB_ID`, -because it changes across the benchmark's walltime-resume chain. +re-exports. The observed 0.2.4 launcher emits +`export NEL_INVOCATION_ID="${NEL_INVOCATION_ID}"` without first assigning it, then +exits with `NEL_INVOCATION_ID: unbound variable` under `set -u` before the evaluation +client starts. In a dry-run, verify that a literal assignment appears before the +re-export; do not substitute `SLURM_JOB_ID`, because it changes across the +benchmark's walltime-resume chain. ## Where each piece runs diff --git a/plugins/modelopt/skills/evaluation/scripts/nel-gdpval.sh b/plugins/modelopt/skills/evaluation/scripts/nel-gdpval.sh index 3c91fd9dbd4..7174495308c 100755 --- a/plugins/modelopt/skills/evaluation/scripts/nel-gdpval.sh +++ b/plugins/modelopt/skills/evaluation/scripts/nel-gdpval.sh @@ -15,11 +15,11 @@ # limitations under the License. # GDPVal requires launcher 0.2.6. In particular, 0.2.6 assigns the generated -# NEL_INVOCATION_ID in run.sub before evaluation.env_vars re-exports it. Older -# launchers can instead fail under `set -u` before the evaluation client starts. +# NEL_INVOCATION_ID in run.sub before evaluation.env_vars re-exports it. The +# observed 0.2.4 launcher instead fails under `set -u` before client startup. set -euo pipefail -NEL_GDPVAL_SPEC="${NEL_GDPVAL_SPEC:-nemo-evaluator-launcher[all]==0.2.6}" +readonly NEL_GDPVAL_SPEC="nemo-evaluator-launcher[all]==0.2.6" case "${1:-}" in -h|--help) diff --git a/plugins/modelopt/skills/evaluation/tests/test_nel_gdpval.py b/plugins/modelopt/skills/evaluation/tests/test_nel_gdpval.py new file mode 100644 index 00000000000..3728047bbd6 --- /dev/null +++ b/plugins/modelopt/skills/evaluation/tests/test_nel_gdpval.py @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +from pathlib import Path + +SCRIPT = Path(__file__).parents[1] / "scripts" / "nel-gdpval.sh" + + +def test_launcher_uses_exact_pin_despite_environment_override(tmp_path): + args_file = tmp_path / "uvx-args" + uvx = tmp_path / "uvx" + uvx.write_text('#!/usr/bin/env bash\nprintf "%s\\n" "$@" > "$UVX_ARGS_FILE"\n') + uvx.chmod(0o755) + + env = os.environ.copy() + env.update( + { + "NEL_GDPVAL_SPEC": "nemo-evaluator-launcher[all]==0.2.4", + "PATH": f"{tmp_path}:{env['PATH']}", + "UVX_ARGS_FILE": str(args_file), + } + ) + + subprocess.run([SCRIPT, "run", "--config", "gdpval.yaml"], env=env, check=True) + + assert args_file.read_text().splitlines() == [ + "--python", + "3.10", + "--from", + "nemo-evaluator-launcher[all]==0.2.6", + "nel", + "run", + "--config", + "gdpval.yaml", + ] From 6948c989faa302c8af571ef8719d092b0b4d0cf4 Mon Sep 17 00:00:00 2001 From: Zhiyu Cheng Date: Thu, 13 Aug 2026 16:36:26 -0700 Subject: [PATCH 3/3] Document GDPVal launcher pin updates Signed-off-by: Zhiyu Cheng --- plugins/modelopt/skills/evaluation/SKILL.md | 18 +++---- .../gym_gdpval/example_gym_gdpval.yaml | 2 +- .../evaluation/recipes/tasks/aa_gym/gdpval.md | 15 +++--- .../evaluation/references/gym-gdpval.md | 48 ++++++++++++++----- .../skills/evaluation/scripts/nel-gdpval.sh | 7 ++- .../evaluation/tests/test_nel_gdpval.py | 5 +- 6 files changed, 60 insertions(+), 35 deletions(-) diff --git a/plugins/modelopt/skills/evaluation/SKILL.md b/plugins/modelopt/skills/evaluation/SKILL.md index 8940264b819..07cb120bd0d 100644 --- a/plugins/modelopt/skills/evaluation/SKILL.md +++ b/plugins/modelopt/skills/evaluation/SKILL.md @@ -34,8 +34,8 @@ If `MODELOPT_WORKSPACE_ROOT` is set, use the common skill's `workspace-managemen ### nel-next path (Terminal-Bench 2.x, SWE-bench, …) — branch here FIRST -A few **agentic** AA benchmarks do **not** run on the default -`nemo-evaluator-launcher` 0.2.6 (Steps 1–9 don't apply). They run on **nel-next** +A few **agentic** AA benchmarks do **not** run on the currently validated +`nemo-evaluator-launcher` 0.2.6 path (Steps 1–9 don't apply). They run on **nel-next** (`nemo-evaluator[harbor]` 0.4.x) — a separate package, CLI (`nel eval run`), `-O` overrides, and `services`/`benchmarks`/`cluster`/`output` schema. If the user asks for one, do **not** add it to a 0.2.6 `evaluation.tasks` list — instead: @@ -45,16 +45,16 @@ for one, do **not** add it to a 0.2.6 `evaluation.tasks` list — instead: 3. Run **`modelopttools:eval-config`** (Step 3b) to write the AWS-sandbox creds + harbor infra rows (`${NEL_NEXT_EVAL_IMAGE}`, `${HARBOR_*_ECR_REPOSITORY}`) into `.env`; always include the `output.export_config.mlflow` block. 4. Dry-run → canary → full (`nel-next.sh eval run`), then **push to MLflow** — SLURM doesn't auto-export, so run `nel-next.sh mlflow-push -r -c ` after (config-driven; see `references/nel-next.md`). -Steps 1–9 below are the 0.2.6 path — use them for everything else. +Steps 1–9 below are currently validated with 0.2.6 — use them for everything else. --- ### GDPVal (NeMo Gym "Stirrup" agent) path — branch here too -GDPVal **does** run on the 0.2.6 `nel` launcher (as a `nemo_gym` task, not -nel-next), so Steps 1–9 apply — but it is mechanically special and **standalone** -(one gym eval per config; never mix it with `aa/` tasks). If the user asks for -GDPVal: +GDPVal **does** run on the currently validated 0.2.6 `nel` launcher (as a +`nemo_gym` task, not nel-next), so Steps 1–9 apply — but it is mechanically +special and **standalone** (one gym eval per config; never mix it with `aa/` +tasks). If the user asks for GDPVal: 1. Read **`references/gym-gdpval.md`** (Apptainer SIF sandbox, gym prepare/reap machinery, deploy sizing, rubric-vs-comparison scoring, MLflow deliverables trap, @@ -71,8 +71,8 @@ GDPVal: `INFERENCE_JUDGE_URL`, `GDPVAL_SIF_DIR`, and `NEMO_EVALUATOR_TRUST_PRE_CMD=1` (the config has a `pre_cmd`). Thinking mode is mandatory (non-thinking loses ~86%). 4. Run both dry-run and launch through `"$SKILL_DIR/scripts/nel-gdpval.sh"`; it - enforces the required 0.2.6 launcher even if `nel` on PATH is stale and avoids - the observed 0.2.4 launcher's unset `NEL_INVOCATION_ID` failure. + enforces the currently validated 0.2.6 launcher even if `nel` on PATH is stale + and avoids an unset `NEL_INVOCATION_ID` failure before client startup. **`limit_samples` is inert on the gym path** (the gym runs all 220 tasks regardless), so there is no cheap canary: watch the real run's first ~20–30 min for the SIF-sandbox line and judge auth, and cancel if wrong. See the diff --git a/plugins/modelopt/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml b/plugins/modelopt/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml index 8d3f1811ca7..199739bef08 100644 --- a/plugins/modelopt/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml +++ b/plugins/modelopt/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml @@ -144,7 +144,7 @@ evaluation: # deliverables are NOT auto-uploaded — they stay on disk for inspection. # Drop "_cache" if you WANT them uploaded as artifacts. PERSIST_DELIVERABLES_DIR: lit:/results/gdpval/deliverables_cache - # Requires launcher 0.2.6 via $SKILL_DIR/scripts/nel-gdpval.sh. That launcher + # Use the currently validated launcher via $SKILL_DIR/scripts/nel-gdpval.sh. It # assigns this stable ID before the runtime re-export (including across resumes). NEL_INVOCATION_ID: runtime:NEL_INVOCATION_ID # Installs apptainer + squashfuse into the eval container (needs diff --git a/plugins/modelopt/skills/evaluation/recipes/tasks/aa_gym/gdpval.md b/plugins/modelopt/skills/evaluation/recipes/tasks/aa_gym/gdpval.md index e7f838ef679..213da448a7e 100644 --- a/plugins/modelopt/skills/evaluation/recipes/tasks/aa_gym/gdpval.md +++ b/plugins/modelopt/skills/evaluation/recipes/tasks/aa_gym/gdpval.md @@ -12,13 +12,14 @@ deliverables inside a per-task Apptainer code-exec sandbox, then a pairwise/rubr judge (**Gemini 3.1 Pro**) scores them. It is the most resource-intensive benchmark in the suite — **220 tasks**, `num_repeats=1`, 4 judge trials per rollout. -It runs on the **0.2.6 `nel` launcher** as a `nemo_gym` task (NOT nel-next), so -Steps 1–9 apply — but with the branch differences below. - -Run it through `"$SKILL_DIR/scripts/nel-gdpval.sh"`, which pins the launcher to -0.2.6. Do not use an unversioned `nel` from PATH: the observed 0.2.4 launcher fails -before client startup when the config forwards `NEL_INVOCATION_ID`. The shared -reference explains the failure signature and dry-run check. +It is currently validated with the **0.2.6 `nel` launcher** as a `nemo_gym` task +(NOT nel-next), so Steps 1–9 apply — but with the branch differences below. + +Run it through `"$SKILL_DIR/scripts/nel-gdpval.sh"`, which pins the currently +validated launcher. Do not use an unversioned `nel` from PATH: an incompatible +launcher can fail before client startup when the config forwards +`NEL_INVOCATION_ID`. The shared reference explains the failure signature, dry-run +check, and procedure for validating and adopting a newer launcher. ## What makes GDPVal different (not a normal `aa/` task) diff --git a/plugins/modelopt/skills/evaluation/references/gym-gdpval.md b/plugins/modelopt/skills/evaluation/references/gym-gdpval.md index 6cd3ee279de..83adf10dd7f 100644 --- a/plugins/modelopt/skills/evaluation/references/gym-gdpval.md +++ b/plugins/modelopt/skills/evaluation/references/gym-gdpval.md @@ -1,13 +1,13 @@ # GDPVal (NeMo Gym "Stirrup" agent) — reference for the gym / agentic path -GDPVal runs on the **0.2.6 `nel` launcher** as a `nemo_gym` task, but it is -mechanically unlike the `aa/` nemo-skills tasks: the Stirrup agent produces -office/PDF **deliverables** in a per-task **Apptainer** code-exec sandbox, a -pairwise/rubric **judge** (Gemini 3.1 Pro) scores them, and NeMo Gym is pulled and -run **inline in the eval container** (`install_on_the_fly`) via `ng_prepare_benchmark` -+ `ng_e2e_collect_rollouts`. This file is the shared machinery; the config template -is `recipes/examples/gym_gdpval/` and the per-task pointer is -`recipes/tasks/aa_gym/gdpval.md`. +GDPVal is currently validated with the **0.2.6 `nel` launcher** as a `nemo_gym` +task, but it is mechanically unlike the `aa/` nemo-skills tasks: the Stirrup agent +produces office/PDF **deliverables** in a per-task **Apptainer** code-exec sandbox, +a pairwise/rubric **judge** (Gemini 3.1 Pro) scores them, and NeMo Gym is pulled +and run **inline in the eval container** (`install_on_the_fly`) via +`ng_prepare_benchmark` + `ng_e2e_collect_rollouts`. This file is the shared +machinery; the config template is `recipes/examples/gym_gdpval/` and the per-task +pointer is `recipes/tasks/aa_gym/gdpval.md`. Always invoke GDPVal through the pinned wrapper, even if `nel` is already on PATH: @@ -17,15 +17,39 @@ Always invoke GDPVal through the pinned wrapper, even if `nel` is already on PAT "$SKILL_DIR/scripts/nel-gdpval.sh" run --config ``` -This is a correctness requirement, not only a dependency pin. Launcher 0.2.6 writes -the generated `NEL_INVOCATION_ID` into `run.sub` before environment-variable -re-exports. The observed 0.2.4 launcher emits +Using the wrapper with a validated pin is a correctness and reproducibility +requirement. The currently pinned 0.2.6 launcher writes the generated +`NEL_INVOCATION_ID` into `run.sub` before environment-variable re-exports. The +failure this pin fixes emits `export NEL_INVOCATION_ID="${NEL_INVOCATION_ID}"` without first assigning it, then exits with `NEL_INVOCATION_ID: unbound variable` under `set -u` before the evaluation client starts. In a dry-run, verify that a literal assignment appears before the re-export; do not substitute `SLURM_JOB_ID`, because it changes across the benchmark's walltime-resume chain. +The exact version is a reproducibility baseline, not a claim that future launchers +are incompatible. Keep baseline and candidate evaluations on the same validated +launcher so a harness change does not become part of the measured model delta. + +## Updating the launcher pin + +When a newer `nemo-evaluator-launcher` release is available: + +1. Review its release notes for launcher schema, generated Slurm, resume, and export + changes. +2. Update `NEL_GDPVAL_VERSION` in `scripts/nel-gdpval.sh` and the expected spec in + `tests/test_nel_gdpval.py`. +3. Run the focused test and pre-commit checks. Verify `nel-gdpval.sh --version` + reports the candidate version. +4. Dry-run a known GDPVal config and confirm the literal `NEL_INVOCATION_ID` + assignment still precedes its runtime re-export in every generated `run.sub`. +5. Launch with the candidate version and monitor the first 20–30 minutes for SIF + sandbox startup and judge authentication. GDPVal ignores `limit_samples`, so + there is no cheap reduced-sample canary. + +Only then update the validated version used for scored runs. Do not mix launcher +versions within a baseline-versus-candidate comparison. + ## Where each piece runs | Component | Where | @@ -201,7 +225,7 @@ mount source, and `raise ValueError` listing the missing ones **before** any | `GDPVAL_REF_FILES_DIR` | lit:/gdpval_ref_files | shared-FS ref-file staging (node-local /tmp breaks multi-node Ray) | | `PERSIST_DELIVERABLES_DIR` | lit | where deliverables persist (see MLflow note) | | `GDPVAL_MAX_TURNS` | lit (optional) | Stirrup turn cap (default 100; golden uses 250) | -| `NEL_INVOCATION_ID` | runtime | stable run id assigned by launcher 0.2.6; do not use `SLURM_JOB_ID` | +| `NEL_INVOCATION_ID` | runtime | stable run id assigned by the validated launcher; do not use `SLURM_JOB_ID` | `INFERENCE_JUDGE_URL` is the judge host — config (from `.env`), substituted as the literal `` placeholder in `gdpval_judge.base_url`, **not** diff --git a/plugins/modelopt/skills/evaluation/scripts/nel-gdpval.sh b/plugins/modelopt/skills/evaluation/scripts/nel-gdpval.sh index 7174495308c..64d8fe2c3ee 100755 --- a/plugins/modelopt/skills/evaluation/scripts/nel-gdpval.sh +++ b/plugins/modelopt/skills/evaluation/scripts/nel-gdpval.sh @@ -14,12 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -# GDPVal requires launcher 0.2.6. In particular, 0.2.6 assigns the generated -# NEL_INVOCATION_ID in run.sub before evaluation.env_vars re-exports it. The -# observed 0.2.4 launcher instead fails under `set -u` before client startup. set -euo pipefail -readonly NEL_GDPVAL_SPEC="nemo-evaluator-launcher[all]==0.2.6" +# Exact known-good pin; follow references/gym-gdpval.md before bumping it. +readonly NEL_GDPVAL_VERSION="0.2.6" +readonly NEL_GDPVAL_SPEC="nemo-evaluator-launcher[all]==${NEL_GDPVAL_VERSION}" case "${1:-}" in -h|--help) diff --git a/plugins/modelopt/skills/evaluation/tests/test_nel_gdpval.py b/plugins/modelopt/skills/evaluation/tests/test_nel_gdpval.py index 3728047bbd6..ce9e367a058 100644 --- a/plugins/modelopt/skills/evaluation/tests/test_nel_gdpval.py +++ b/plugins/modelopt/skills/evaluation/tests/test_nel_gdpval.py @@ -20,7 +20,7 @@ SCRIPT = Path(__file__).parents[1] / "scripts" / "nel-gdpval.sh" -def test_launcher_uses_exact_pin_despite_environment_override(tmp_path): +def test_launcher_uses_validated_pin_despite_environment_override(tmp_path): args_file = tmp_path / "uvx-args" uvx = tmp_path / "uvx" uvx.write_text('#!/usr/bin/env bash\nprintf "%s\\n" "$@" > "$UVX_ARGS_FILE"\n') @@ -29,7 +29,8 @@ def test_launcher_uses_exact_pin_despite_environment_override(tmp_path): env = os.environ.copy() env.update( { - "NEL_GDPVAL_SPEC": "nemo-evaluator-launcher[all]==0.2.4", + "NEL_GDPVAL_SPEC": "nemo-evaluator-launcher[all]==0.0.0", + "NEL_GDPVAL_VERSION": "0.0.0", "PATH": f"{tmp_path}:{env['PATH']}", "UVX_ARGS_FILE": str(args_file), }