From 47591f5ecbae23cbf7ca8d13cc5ba45d1f6a4d4c Mon Sep 17 00:00:00 2001 From: Yushu Li Date: Sat, 31 Oct 2026 20:22:00 -0700 Subject: [PATCH] fix(evaluator): preserve boolean flags through cascade evaluation --- openevolve/evaluator.py | 12 +-- tests/test_cascade_boolean_metrics.py | 102 ++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 6 deletions(-) create mode 100644 tests/test_cascade_boolean_metrics.py diff --git a/openevolve/evaluator.py b/openevolve/evaluator.py index 084d1c04c..e9bb7de67 100644 --- a/openevolve/evaluator.py +++ b/openevolve/evaluator.py @@ -575,14 +575,14 @@ async def run_stage2(): # Merge results from stage 1 and 2 merged_metrics = {} - # Convert all values to float to avoid type errors + # Normalize numeric scores while preserving boolean flags as metadata for name, value in stage1_eval_result.metrics.items(): if isinstance(value, (int, float)) and name != "error": - merged_metrics[name] = float(value) + merged_metrics[name] = value if isinstance(value, bool) else float(value) for name, value in stage2_eval_result.metrics.items(): if isinstance(value, (int, float)) and name != "error": - merged_metrics[name] = float(value) + merged_metrics[name] = value if isinstance(value, bool) else float(value) # Merge artifacts merged_artifacts = {} @@ -640,7 +640,7 @@ async def run_stage3(): # Merge stage 3 results for name, value in stage3_eval_result.metrics.items(): if isinstance(value, (int, float)) and name != "error": - merged_result.metrics[name] = float(value) + merged_result.metrics[name] = value if isinstance(value, bool) else float(value) merged_result.artifacts.update(stage3_eval_result.artifacts) @@ -808,8 +808,8 @@ def _passes_threshold(self, metrics: Dict[str, float], threshold: float) -> bool # This maintains backward compatibility valid_metrics = [] for name, value in metrics.items(): - # Skip 'error' keys and ensure values are numeric - if name != "error" and isinstance(value, (int, float)): + # Flags are metadata, not scores (bool is a subclass of int). + if name != "error" and isinstance(value, (int, float)) and not isinstance(value, bool): try: valid_metrics.append(float(value)) except (TypeError, ValueError): diff --git a/tests/test_cascade_boolean_metrics.py b/tests/test_cascade_boolean_metrics.py new file mode 100644 index 000000000..34e445b77 --- /dev/null +++ b/tests/test_cascade_boolean_metrics.py @@ -0,0 +1,102 @@ +"""Boolean flags must remain metadata throughout successful cascade evaluations.""" + +import sys +import tempfile +import unittest +from pathlib import Path + +from openevolve.config import EvaluatorConfig +from openevolve.evaluator import Evaluator +from openevolve.utils.metrics_utils import get_fitness_score + + +class TestCascadeBooleanMetrics(unittest.IsolatedAsyncioTestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.original_path = sys.path[:] + self.addCleanup(lambda: sys.path.__setitem__(slice(None), self.original_path)) + self.original_module = sys.modules.get("evaluation_module") + self.addCleanup(self.restore_module) + + def restore_module(self): + if self.original_module is None: + sys.modules.pop("evaluation_module", None) + else: + sys.modules["evaluation_module"] = self.original_module + + def make_evaluator(self, stages, thresholds=(0.5, 0.5)): + path = Path(self.directory.name) / "evaluator.py" + source = "def evaluate(path):\n return {}\n" + for i, metrics in enumerate(stages, 1): + source += f"def evaluate_stage{i}(path):\n return {metrics!r}\n" + path.write_text(source, encoding="utf-8") + evaluator = Evaluator( + EvaluatorConfig(cascade_evaluation=True, cascade_thresholds=list(thresholds)), + str(path), + ) + self.addCleanup(evaluator._executor.shutdown, wait=True) + return evaluator + + async def test_true_flag_does_not_advance_a_low_scoring_first_stage(self): + first = {"score": 0.1, "valid": True} + evaluator = self.make_evaluator([first, {"score": 0.9, "later_stage": 1.0}]) + result = await evaluator._cascade_evaluate("unused.py") + self.assertEqual(result.metrics, first) + + async def test_false_flag_does_not_block_a_high_scoring_first_stage(self): + evaluator = self.make_evaluator( + [{"score": 0.8, "flag": False}, {"score": 0.9, "later_stage": 1.0}] + ) + result = await evaluator._cascade_evaluate("unused.py") + self.assertEqual(result.metrics["later_stage"], 1.0) + + async def test_true_flags_do_not_advance_a_low_scoring_second_stage(self): + evaluator = self.make_evaluator( + [{"score": 0.8, "valid": True}, {"score": 0.1}, {"score": 0.9, "third": 1.0}] + ) + result = await evaluator._cascade_evaluate("unused.py") + self.assertEqual(result.metrics["score"], 0.1) + self.assertNotIn("third", result.metrics) + + async def test_merges_preserve_flags_and_fallback_fitness(self): + for count in (2, 3): + with self.subTest(stages=count): + stages = [ + {"score": 0.2, f"valid_{i}": True, f"optional_{i}": False} for i in range(count) + ] + evaluator = self.make_evaluator(stages, thresholds=(0.0, 0.0)) + result = await evaluator._cascade_evaluate("unused.py") + self.assertAlmostEqual(get_fitness_score(result.metrics), 0.2) + for i in range(count): + self.assertIs(result.metrics[f"valid_{i}"], True) + self.assertIs(result.metrics[f"optional_{i}"], False) + + def test_flag_only_metrics_do_not_pass_even_a_zero_threshold(self): + evaluator = self.make_evaluator([{}, {}]) + for metrics in ({"valid": True}, {"valid": False}, {"error": 0.0, "valid": True}): + with self.subTest(metrics=metrics): + self.assertFalse(evaluator._passes_threshold(metrics, 0.0)) + + def test_numeric_combined_score_still_takes_precedence(self): + evaluator = self.make_evaluator([{}, {}]) + for combined, other, expected in ((0.0, 1.0, False), (0.5, 0.0, True)): + with self.subTest(combined=combined): + self.assertEqual( + evaluator._passes_threshold( + {"combined_score": combined, "score": other, "flag": True}, 0.5 + ), + expected, + ) + + async def test_numeric_scores_and_error_filtering_are_unchanged(self): + evaluator = self.make_evaluator( + [{"score": 1, "error": 0.0}, {"score": 0.8}, {"score": 0.9}] + ) + result = await evaluator._cascade_evaluate("unused.py") + self.assertEqual(result.metrics, {"score": 0.9}) + self.assertIsInstance(result.metrics["score"], float) + + +if __name__ == "__main__": + unittest.main()