diff --git a/plugins/modelopt/skills/evaluation/SKILL.md b/plugins/modelopt/skills/evaluation/SKILL.md index c0716a71d4d..07cb120bd0d 100644 --- a/plugins/modelopt/skills/evaluation/SKILL.md +++ b/plugins/modelopt/skills/evaluation/SKILL.md @@ -34,8 +34,8 @@ If `MODELOPT_WORKSPACE_ROOT` is set, use the common skill's `workspace-managemen ### nel-next path (Terminal-Bench 2.x, SWE-bench, …) — branch here FIRST -A few **agentic** AA benchmarks do **not** run on the default -`nemo-evaluator-launcher` 0.2.6 (Steps 1–9 don't apply). They run on **nel-next** +A few **agentic** AA benchmarks do **not** run on the currently validated +`nemo-evaluator-launcher` 0.2.6 path (Steps 1–9 don't apply). They run on **nel-next** (`nemo-evaluator[harbor]` 0.4.x) — a separate package, CLI (`nel eval run`), `-O` overrides, and `services`/`benchmarks`/`cluster`/`output` schema. If the user asks for one, do **not** add it to a 0.2.6 `evaluation.tasks` list — instead: @@ -45,16 +45,16 @@ for one, do **not** add it to a 0.2.6 `evaluation.tasks` list — instead: 3. Run **`modelopttools:eval-config`** (Step 3b) to write the AWS-sandbox creds + harbor infra rows (`${NEL_NEXT_EVAL_IMAGE}`, `${HARBOR_*_ECR_REPOSITORY}`) into `.env`; always include the `output.export_config.mlflow` block. 4. Dry-run → canary → full (`nel-next.sh eval run`), then **push to MLflow** — SLURM doesn't auto-export, so run `nel-next.sh mlflow-push -r -c ` after (config-driven; see `references/nel-next.md`). -Steps 1–9 below are the 0.2.6 path — use them for everything else. +Steps 1–9 below are currently validated with 0.2.6 — use them for everything else. --- ### GDPVal (NeMo Gym "Stirrup" agent) path — branch here too -GDPVal **does** run on the 0.2.6 `nel` launcher (as a `nemo_gym` task, not -nel-next), so Steps 1–9 apply — but it is mechanically special and **standalone** -(one gym eval per config; never mix it with `aa/` tasks). If the user asks for -GDPVal: +GDPVal **does** run on the currently validated 0.2.6 `nel` launcher (as a +`nemo_gym` task, not nel-next), so Steps 1–9 apply — but it is mechanically +special and **standalone** (one gym eval per config; never mix it with `aa/` +tasks). If the user asks for GDPVal: 1. Read **`references/gym-gdpval.md`** (Apptainer SIF sandbox, gym prepare/reap machinery, deploy sizing, rubric-vs-comparison scoring, MLflow deliverables trap, @@ -70,8 +70,11 @@ GDPVal: unsandboxed. Verify with `gdpval-sif.sh --check`. `.env` needs `HF_TOKEN`, `INFERENCE_API_KEY`, `TAVILY_API_KEY`, `INFERENCE_JUDGE_URL`, `GDPVAL_SIF_DIR`, and `NEMO_EVALUATOR_TRUST_PRE_CMD=1` (the config has a `pre_cmd`). Thinking mode is mandatory (non-thinking loses ~86%). -4. Dry-run → launch. **`limit_samples` is inert on the gym path** (the gym runs all - 220 tasks regardless), so there is no cheap canary: watch the real run's first +4. Run both dry-run and launch through `"$SKILL_DIR/scripts/nel-gdpval.sh"`; it + enforces the currently validated 0.2.6 launcher even if `nel` on PATH is stale + and avoids an unset `NEL_INVOCATION_ID` failure before client startup. + **`limit_samples` is inert on the gym path** (the gym runs all 220 tasks + regardless), so there is no cheap canary: watch the real run's first ~20–30 min for the SIF-sandbox line and judge auth, and cancel if wrong. See the recipe's Canary section. diff --git a/plugins/modelopt/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml b/plugins/modelopt/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml index cdc513dc8cd..199739bef08 100644 --- a/plugins/modelopt/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml +++ b/plugins/modelopt/skills/evaluation/recipes/examples/gym_gdpval/example_gym_gdpval.yaml @@ -144,6 +144,8 @@ evaluation: # deliverables are NOT auto-uploaded — they stay on disk for inspection. # Drop "_cache" if you WANT them uploaded as artifacts. PERSIST_DELIVERABLES_DIR: lit:/results/gdpval/deliverables_cache + # Use the currently validated launcher via $SKILL_DIR/scripts/nel-gdpval.sh. It + # assigns this stable ID before the runtime re-export (including across resumes). NEL_INVOCATION_ID: runtime:NEL_INVOCATION_ID # Installs apptainer + squashfuse into the eval container (needs # NEMO_EVALUATOR_TRUST_PRE_CMD=1 in the launching shell). See references/gym-gdpval.md. diff --git a/plugins/modelopt/skills/evaluation/recipes/tasks/aa_gym/gdpval.md b/plugins/modelopt/skills/evaluation/recipes/tasks/aa_gym/gdpval.md index 13a83562f82..213da448a7e 100644 --- a/plugins/modelopt/skills/evaluation/recipes/tasks/aa_gym/gdpval.md +++ b/plugins/modelopt/skills/evaluation/recipes/tasks/aa_gym/gdpval.md @@ -12,8 +12,14 @@ deliverables inside a per-task Apptainer code-exec sandbox, then a pairwise/rubr judge (**Gemini 3.1 Pro**) scores them. It is the most resource-intensive benchmark in the suite — **220 tasks**, `num_repeats=1`, 4 judge trials per rollout. -It runs on the **0.2.6 `nel` launcher** as a `nemo_gym` task (NOT nel-next), so -Steps 1–9 apply — but with the branch differences below. +It is currently validated with the **0.2.6 `nel` launcher** as a `nemo_gym` task +(NOT nel-next), so Steps 1–9 apply — but with the branch differences below. + +Run it through `"$SKILL_DIR/scripts/nel-gdpval.sh"`, which pins the currently +validated launcher. Do not use an unversioned `nel` from PATH: an incompatible +launcher can fail before client startup when the config forwards +`NEL_INVOCATION_ID`. The shared reference explains the failure signature, dry-run +check, and procedure for validating and adopting a newer launcher. ## What makes GDPVal different (not a normal `aa/` task) diff --git a/plugins/modelopt/skills/evaluation/references/gym-gdpval.md b/plugins/modelopt/skills/evaluation/references/gym-gdpval.md index 61aaf0746f7..83adf10dd7f 100644 --- a/plugins/modelopt/skills/evaluation/references/gym-gdpval.md +++ b/plugins/modelopt/skills/evaluation/references/gym-gdpval.md @@ -1,13 +1,54 @@ # GDPVal (NeMo Gym "Stirrup" agent) — reference for the gym / agentic path -GDPVal runs on the **0.2.6 `nel` launcher** as a `nemo_gym` task, but it is -mechanically unlike the `aa/` nemo-skills tasks: the Stirrup agent produces -office/PDF **deliverables** in a per-task **Apptainer** code-exec sandbox, a -pairwise/rubric **judge** (Gemini 3.1 Pro) scores them, and NeMo Gym is pulled and -run **inline in the eval container** (`install_on_the_fly`) via `ng_prepare_benchmark` -+ `ng_e2e_collect_rollouts`. This file is the shared machinery; the config template -is `recipes/examples/gym_gdpval/` and the per-task pointer is -`recipes/tasks/aa_gym/gdpval.md`. +GDPVal is currently validated with the **0.2.6 `nel` launcher** as a `nemo_gym` +task, but it is mechanically unlike the `aa/` nemo-skills tasks: the Stirrup agent +produces office/PDF **deliverables** in a per-task **Apptainer** code-exec sandbox, +a pairwise/rubric **judge** (Gemini 3.1 Pro) scores them, and NeMo Gym is pulled +and run **inline in the eval container** (`install_on_the_fly`) via +`ng_prepare_benchmark` + `ng_e2e_collect_rollouts`. This file is the shared +machinery; the config template is `recipes/examples/gym_gdpval/` and the per-task +pointer is `recipes/tasks/aa_gym/gdpval.md`. + +Always invoke GDPVal through the pinned wrapper, even if `nel` is already on PATH: + +```bash +"$SKILL_DIR/scripts/nel-gdpval.sh" --version # must report nemo_evaluator_launcher: 0.2.6 +"$SKILL_DIR/scripts/nel-gdpval.sh" run --config --dry-run +"$SKILL_DIR/scripts/nel-gdpval.sh" run --config +``` + +Using the wrapper with a validated pin is a correctness and reproducibility +requirement. The currently pinned 0.2.6 launcher writes the generated +`NEL_INVOCATION_ID` into `run.sub` before environment-variable re-exports. The +failure this pin fixes emits +`export NEL_INVOCATION_ID="${NEL_INVOCATION_ID}"` without first assigning it, then +exits with `NEL_INVOCATION_ID: unbound variable` under `set -u` before the evaluation +client starts. In a dry-run, verify that a literal assignment appears before the +re-export; do not substitute `SLURM_JOB_ID`, because it changes across the +benchmark's walltime-resume chain. + +The exact version is a reproducibility baseline, not a claim that future launchers +are incompatible. Keep baseline and candidate evaluations on the same validated +launcher so a harness change does not become part of the measured model delta. + +## Updating the launcher pin + +When a newer `nemo-evaluator-launcher` release is available: + +1. Review its release notes for launcher schema, generated Slurm, resume, and export + changes. +2. Update `NEL_GDPVAL_VERSION` in `scripts/nel-gdpval.sh` and the expected spec in + `tests/test_nel_gdpval.py`. +3. Run the focused test and pre-commit checks. Verify `nel-gdpval.sh --version` + reports the candidate version. +4. Dry-run a known GDPVal config and confirm the literal `NEL_INVOCATION_ID` + assignment still precedes its runtime re-export in every generated `run.sub`. +5. Launch with the candidate version and monitor the first 20–30 minutes for SIF + sandbox startup and judge authentication. GDPVal ignores `limit_samples`, so + there is no cheap reduced-sample canary. + +Only then update the validated version used for scored runs. Do not mix launcher +versions within a baseline-versus-candidate comparison. ## Where each piece runs @@ -184,7 +225,7 @@ mount source, and `raise ValueError` listing the missing ones **before** any | `GDPVAL_REF_FILES_DIR` | lit:/gdpval_ref_files | shared-FS ref-file staging (node-local /tmp breaks multi-node Ray) | | `PERSIST_DELIVERABLES_DIR` | lit | where deliverables persist (see MLflow note) | | `GDPVAL_MAX_TURNS` | lit (optional) | Stirrup turn cap (default 100; golden uses 250) | -| `NEL_INVOCATION_ID` | runtime | run id | +| `NEL_INVOCATION_ID` | runtime | stable run id assigned by the validated launcher; do not use `SLURM_JOB_ID` | `INFERENCE_JUDGE_URL` is the judge host — config (from `.env`), substituted as the literal `` placeholder in `gdpval_judge.base_url`, **not** diff --git a/plugins/modelopt/skills/evaluation/scripts/nel-gdpval.sh b/plugins/modelopt/skills/evaluation/scripts/nel-gdpval.sh new file mode 100755 index 00000000000..64d8fe2c3ee --- /dev/null +++ b/plugins/modelopt/skills/evaluation/scripts/nel-gdpval.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +# Exact known-good pin; follow references/gym-gdpval.md before bumping it. +readonly NEL_GDPVAL_VERSION="0.2.6" +readonly NEL_GDPVAL_SPEC="nemo-evaluator-launcher[all]==${NEL_GDPVAL_VERSION}" + +case "${1:-}" in + -h|--help) + echo "usage: nel-gdpval.sh " + echo " nel-gdpval.sh --version" + exit 0 + ;; +esac + +command -v uvx >/dev/null 2>&1 || { + echo "ERROR: 'uvx' is required to run the pinned GDPVal launcher" >&2 + exit 1 +} + +exec uvx --python 3.10 --from "$NEL_GDPVAL_SPEC" nel "$@" diff --git a/plugins/modelopt/skills/evaluation/tests/test_nel_gdpval.py b/plugins/modelopt/skills/evaluation/tests/test_nel_gdpval.py new file mode 100644 index 00000000000..ce9e367a058 --- /dev/null +++ b/plugins/modelopt/skills/evaluation/tests/test_nel_gdpval.py @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +from pathlib import Path + +SCRIPT = Path(__file__).parents[1] / "scripts" / "nel-gdpval.sh" + + +def test_launcher_uses_validated_pin_despite_environment_override(tmp_path): + args_file = tmp_path / "uvx-args" + uvx = tmp_path / "uvx" + uvx.write_text('#!/usr/bin/env bash\nprintf "%s\\n" "$@" > "$UVX_ARGS_FILE"\n') + uvx.chmod(0o755) + + env = os.environ.copy() + env.update( + { + "NEL_GDPVAL_SPEC": "nemo-evaluator-launcher[all]==0.0.0", + "NEL_GDPVAL_VERSION": "0.0.0", + "PATH": f"{tmp_path}:{env['PATH']}", + "UVX_ARGS_FILE": str(args_file), + } + ) + + subprocess.run([SCRIPT, "run", "--config", "gdpval.yaml"], env=env, check=True) + + assert args_file.read_text().splitlines() == [ + "--python", + "3.10", + "--from", + "nemo-evaluator-launcher[all]==0.2.6", + "nel", + "run", + "--config", + "gdpval.yaml", + ]