diff --git a/openevolve/evaluator.py b/openevolve/evaluator.py index e9bb7de67..f2504c84d 100644 --- a/openevolve/evaluator.py +++ b/openevolve/evaluator.py @@ -8,6 +8,7 @@ import inspect import json import logging +import math import os import subprocess import sys @@ -729,12 +730,24 @@ async def _llm_evaluate(self, program_code: str, program_id: str = "") -> Dict[s continue for name, value in result.items(): - if not isinstance(value, (int, float)): + # Flags are metadata, and invalid numeric feedback must + # not poison valid scores from this or another judge. + if isinstance(value, bool) or not isinstance(value, (int, float)): artifacts[name] = value continue - weighted_metrics[name] = ( - weighted_metrics.get(name, 0.0) + float(value) * weight - ) + try: + score = float(value) + except (ValueError, TypeError, OverflowError): + logger.warning( + "Skipping invalid LLM judge metric %s from judge %d", name, i + ) + continue + if not math.isfinite(score): + logger.warning( + "Skipping non-finite LLM judge metric %s from judge %d", name, i + ) + continue + weighted_metrics[name] = weighted_metrics.get(name, 0.0) + score * weight metric_weights[name] = metric_weights.get(name, 0.0) + weight # A missing or invalid response is not a zero score. Normalize diff --git a/tests/test_llm_evaluation_ensemble.py b/tests/test_llm_evaluation_ensemble.py index 948b4a13f..ca3ad4114 100644 --- a/tests/test_llm_evaluation_ensemble.py +++ b/tests/test_llm_evaluation_ensemble.py @@ -64,3 +64,48 @@ async def test_valid_feedback_still_contributes_to_combined_fitness(self): self.assertAlmostEqual( metrics["llm_average"], 0.8 * self.evaluator.config.llm_feedback_weight ) + + async def test_nonfinite_metrics_do_not_poison_other_judges(self): + for invalid in ("NaN", "Infinity", "-Infinity", "1e999", "-1e999"): + for invalid_index in (0, 1): + with self.subTest(invalid=invalid, invalid_index=invalid_index): + responses = ['{"quality": 0.8}'] * 2 + responses[invalid_index] = '{"quality": ' + invalid + ', "clarity": 0.6}' + result = await self.evaluate_responses(responses) + self.assertEqual(set(result.metrics), {"quality", "clarity"}) + self.assertAlmostEqual(result.metrics["quality"], 0.8) + self.assertAlmostEqual(result.metrics["clarity"], 0.6) + + async def test_unrepresentable_integer_keeps_other_metrics(self): + response = '{"quality": ' + "1" + "0" * 400 + ', "clarity": 0.6}' + result = await self.evaluate_responses([response, '{"quality": 0.8}']) + self.assertEqual(set(result.metrics), {"quality", "clarity"}) + self.assertAlmostEqual(result.metrics["quality"], 0.8) + self.assertAlmostEqual(result.metrics["clarity"], 0.6) + + async def test_boolean_judge_fields_remain_artifacts(self): + result = await self.evaluate_responses( + ['{"quality": 0.4, "valid": true}', '{"quality": 0.8, "timeout": false}'] + ) + self.assertAlmostEqual(result.metrics["quality"], 0.7) + self.assertEqual(set(result.metrics), {"quality"}) + self.assertEqual(result.artifacts, {"valid": True, "timeout": False}) + + async def test_invalid_numeric_feedback_leaves_measured_fitness_unchanged(self): + self.ensemble.generate_all_with_context = AsyncMock( + return_value=['{"quality": NaN}', '{"quality": Infinity}'] + ) + metrics = await self.evaluator.evaluate_program("x = 1", "nonfinite-judges") + self.assertEqual(metrics, {"combined_score": 0.5}) + + async def test_valid_judge_survives_nonfinite_peer_in_combined_fitness(self): + self.ensemble.generate_all_with_context = AsyncMock( + return_value=['{"quality": NaN}', '{"quality": 0.8}'] + ) + metrics = await self.evaluator.evaluate_program("x = 1", "finite-peer") + self.assertAlmostEqual(metrics["combined_score"], 0.59) + self.assertAlmostEqual(metrics["llm_quality"], 0.08) + + async def test_finite_scores_are_not_clamped(self): + result = await self.evaluate_responses(['{"quality": -2, "clarity": 3}', "{}"]) + self.assertEqual(result.metrics, {"quality": -2.0, "clarity": 3.0})