From b1cf58012920d6f8dc954b611d0045fbd54d554f Mon Sep 17 00:00:00 2001 From: ZJianjin Date: Wed, 23 Sep 2026 12:02:22 +0800 Subject: [PATCH] Adjust task-2-5 scoring --- tasks/task-2-5/README.md | 8 ++++---- tasks/task-2-5/instruction.md | 10 +++++----- tasks/task-2-5/tests/grader.py | 4 ++-- tasks/task-2-5/tests/test.sh | 9 +++++++-- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/tasks/task-2-5/README.md b/tasks/task-2-5/README.md index c3a9c26..8cfb9dc 100644 --- a/tasks/task-2-5/README.md +++ b/tasks/task-2-5/README.md @@ -17,7 +17,7 @@ a Python implementation against a corrected, unpruned starter. The goal is to preserve strong retrieval quality while reducing total query wall time relative to that reference. Quality thresholds are hard gates; passing submissions receive a latency reward that decreases linearly between -30% and 50% of starter wall time. +25% and 35% of starter wall time. ## What This Task Tests @@ -92,9 +92,9 @@ minimums are **0.89** and **0.99** respectively. ### Correctness and Latency Reward The verifier measures the candidate and corrected starter on the same workload -and resource allocation. A latency ratio at or below **0.30** receives reward -`1`; a ratio at or above **0.50** receives reward `0`; values strictly between -those boundaries receive `(0.50 - ratio) / 0.20`. Successful build/run, valid +and resource allocation. A latency ratio at or below **0.25** receives reward +`1`; a ratio at or above **0.35** receives reward `0`; values strictly between +those boundaries receive `(0.35 - ratio) / 0.10`. Successful build/run, valid outputs, and implementation restrictions remain hard requirements. ### Integrity Checks and Final Reward diff --git a/tasks/task-2-5/instruction.md b/tasks/task-2-5/instruction.md index 5e14de3..22fa856 100644 --- a/tasks/task-2-5/instruction.md +++ b/tasks/task-2-5/instruction.md @@ -9,7 +9,7 @@ wall time low. The submission must satisfy the held-out retrieval-quality requirements. Its reward then increases as the wall time for the complete hidden query set falls -from 50% to 30% of the corrected, unpruned starter wall time measured by the +from 35% to 25% of the corrected, unpruned starter wall time measured by the verifier in the same environment. You may replace the starter index or retrieval algorithm as long as the executable interface and output contract are preserved. @@ -174,10 +174,10 @@ NDCG and Recall are hard gates. If either quality gate, build/run behavior, output validation, or the trajectory jailbreak judge fails, reward is `0`. Otherwise latency is scored as follows: -- latency ratio at or below `0.30`: reward `1`; -- latency ratio strictly between `0.30` and `0.50`: reward - `(0.50 - latency ratio) / 0.20`; -- latency ratio at or above `0.50`: reward `0`. +- latency ratio at or below `0.25`: reward `1`; +- latency ratio strictly between `0.25` and `0.35`: reward + `(0.35 - latency ratio) / 0.10`; +- latency ratio at or above `0.35`: reward `0`. Submission-reported lookup counters or internal timing are diagnostic only and do not replace verifier wall-time measurement. diff --git a/tasks/task-2-5/tests/grader.py b/tasks/task-2-5/tests/grader.py index fb3a232..d47f45d 100644 --- a/tasks/task-2-5/tests/grader.py +++ b/tasks/task-2-5/tests/grader.py @@ -15,8 +15,8 @@ EPSILON = 1e-12 MIN_NDCG = 0.89 MIN_RECALL = 0.99 -FULL_REWARD_STARTER_WALL_RATIO = 0.30 -ZERO_REWARD_STARTER_WALL_RATIO = 0.50 +FULL_REWARD_STARTER_WALL_RATIO = 0.25 +ZERO_REWARD_STARTER_WALL_RATIO = 0.35 REWARD_METRIC = "quality_gated_linear_starter_latency" diff --git a/tasks/task-2-5/tests/test.sh b/tasks/task-2-5/tests/test.sh index 10ceb68..ec6e65b 100755 --- a/tasks/task-2-5/tests/test.sh +++ b/tasks/task-2-5/tests/test.sh @@ -51,8 +51,13 @@ else fi if [[ -d /tests && ! -L /tests ]]; then chown -R root:root /tests - chmod 0700 /tests - find /tests/data -type f -exec chmod 0600 {} + + # The submission must read only the hidden query file. Keep every test + # directory traversable but non-listable, and every test file root-only + # before selectively exposing the query input below. + find /tests -type d -exec chmod 0711 {} + + find /tests -type f -exec chmod 0600 {} + + chmod 0700 /tests/test.sh + chmod 0644 /tests/data/hidden_queries.jsonl fi if [[ -d /logs/agent && ! -L /logs/agent ]]; then chown root:root /logs/agent