From 47b315997d782d80b0e3855d73de783372593d84 Mon Sep 17 00:00:00 2001 From: SUPR1M3 Date: Sun, 1 Nov 2026 02:04:39 -0500 Subject: [PATCH 1/5] feat: mix full rewrites into diff-based evolution (#116) Add full_rewrite_probability: with diff_based_evolution on, each iteration asks for a full rewrite instead of SEARCH/REPLACE diffs with this probability. The default 0.0 keeps today's behaviour. - The worker picks the mode per iteration from a generator seeded with (random_seed, iteration), so runs stay reproducible whichever worker runs an iteration, and the ensemble's model picks (#559) are untouched. - The chosen mode drives both prompt building and response parsing, and is recorded on the result (failures included), in program metadata ("evolution_mode"), in the evolution trace, and as the template key passed to log_prompt. - Config validation (from_dict and the controller, for configs built in code): the probability must be in [0, 1], needs diff_based_evolution, and cannot be combined with prompt.programs_as_changes_description. Also add evolution_trace.include_failures (off by default): iterations that fail before producing a child are written to _failures.jsonl with their error and mode, plus the prompt and LLM response when include_prompts is on. Worker errors after the LLM call now carry the prompt, response, parent and island, so parse failures can be inspected. --- configs/default_config.yaml | 5 + openevolve/config.py | 28 ++- openevolve/controller.py | 3 + openevolve/evolution_trace.py | 45 ++++ openevolve/process_parallel.py | 148 +++++++++----- tests/test_controller_trace_failures.py | 144 +++++++++++++ tests/test_diff_format_validation_worker.py | 5 + tests/test_evolution_trace.py | 48 +++++ tests/test_full_rewrite_probability.py | 214 ++++++++++++++++++++ tests/test_worker_config_serialization.py | 1 + 10 files changed, 584 insertions(+), 57 deletions(-) create mode 100644 tests/test_controller_trace_failures.py create mode 100644 tests/test_full_rewrite_probability.py diff --git a/configs/default_config.yaml b/configs/default_config.yaml index 844624a507..1793b90354 100644 --- a/configs/default_config.yaml +++ b/configs/default_config.yaml @@ -11,6 +11,8 @@ random_seed: 42 # Random seed for reproducibility (null = # Evolution settings diff_based_evolution: true # Use diff-based evolution (true) or full rewrites (false) +full_rewrite_probability: 0.0 # With diffs on, chance (0-1) an iteration asks for a full rewrite instead + # (the mode used is saved as metadata["evolution_mode"] on each program) max_code_length: 10000 # Maximum allowed code length in characters enforce_evolve_blocks: false # Revert LLM edits outside EVOLVE-BLOCK-START/END markers @@ -181,6 +183,9 @@ evolution_trace: format: 'jsonl' # Output format: 'jsonl', 'json', or 'hdf5' include_code: false # Include full program code in traces include_prompts: true # Include prompts and LLM responses + include_failures: false # Also log failed iterations (error, mode, and with + # include_prompts the prompt and response) to + # _failures.jsonl output_path: null # Path for trace output (defaults to output_dir/evolution_trace.{format}) buffer_size: 10 # Number of traces to buffer before writing compress: false # Compress output file (jsonl only) diff --git a/openevolve/config.py b/openevolve/config.py index 560dfe0588..61a65c715b 100644 --- a/openevolve/config.py +++ b/openevolve/config.py @@ -445,6 +445,9 @@ class EvolutionTraceConfig: format: str = "jsonl" # Options: "jsonl", "json", "hdf5" include_code: bool = False include_prompts: bool = True + # Also record iterations that failed before a child program existed (unparseable + # response, diff that did not apply, ...) in _failures.jsonl + include_failures: bool = False output_path: Optional[str] = None buffer_size: int = 10 compress: bool = False @@ -475,6 +478,9 @@ class Config: # Evolution settings diff_based_evolution: bool = True + # With diff_based_evolution, the chance that an iteration asks for a full rewrite instead + # of SEARCH/REPLACE diffs (issue #116). 0.0 keeps every iteration diff-based + full_rewrite_probability: float = 0.0 max_code_length: int = 10000 # Revert any LLM edits outside the EVOLVE-BLOCK-START/END regions enforce_evolve_blocks: bool = False @@ -559,14 +565,28 @@ def from_dict(cls, config_dict: Dict[str, Any]) -> "Config": config.database.validate_island_initial_programs() - if config.prompt.programs_as_changes_description and not config.diff_based_evolution: + config.validate_evolution_mode() + + return config + + def validate_evolution_mode(self) -> None: + """Check that diff_based_evolution, full_rewrite_probability and + prompt.programs_as_changes_description agree""" + p = self.full_rewrite_probability + if isinstance(p, bool) or not isinstance(p, (int, float)) or not 0.0 <= p <= 1.0: + raise ValueError(f"full_rewrite_probability must be between 0 and 1, got {p!r}") + if p > 0 and not self.diff_based_evolution: + raise ValueError( + "full_rewrite_probability only applies when diff_based_evolution=true " + "(diff_based_evolution=false already makes every iteration a full rewrite)" + ) + if self.prompt.programs_as_changes_description and (not self.diff_based_evolution or p > 0): raise ValueError( "prompt.programs_as_changes_description=true requires diff_based_evolution=true " - "(full rewrites cannot reliably update code and changes_description together)" + "and full_rewrite_probability=0 (full rewrites cannot reliably update code and " + "changes_description together)" ) - return config - def to_dict(self) -> Dict[str, Any]: return asdict(self) diff --git a/openevolve/controller.py b/openevolve/controller.py index 7977d05c0b..92a1c35bdf 100644 --- a/openevolve/controller.py +++ b/openevolve/controller.py @@ -59,6 +59,8 @@ def __init__( # Load configuration (loaded in main_async) self.config = config self.island_selector = island_selector + # A Config built in code never passes through Config.from_dict, so validate here too + self.config.validate_evolution_mode() # Set up output directory self.output_dir = output_dir or os.path.join( @@ -185,6 +187,7 @@ def __init__( format=self.config.evolution_trace.format, include_code=self.config.evolution_trace.include_code, include_prompts=self.config.evolution_trace.include_prompts, + include_failures=self.config.evolution_trace.include_failures, enabled=True, buffer_size=self.config.evolution_trace.buffer_size, compress=self.config.evolution_trace.compress, diff --git a/openevolve/evolution_trace.py b/openevolve/evolution_trace.py index 8e6699cda3..7f68876bea 100644 --- a/openevolve/evolution_trace.py +++ b/openevolve/evolution_trace.py @@ -73,6 +73,7 @@ def __init__( buffer_size: int = 10, compress: bool = False, include_changes_description: bool = True, + include_failures: bool = False, ): """ Initialize the evolution tracer @@ -86,12 +87,14 @@ def __init__( buffer_size: Number of traces to buffer before writing compress: Whether to compress output files include_changes_description: Whether to include per-program changes descriptions + include_failures: Whether to record failed iterations (see log_failure) """ self.enabled = enabled self.format = format self.include_code = include_code self.include_prompts = include_prompts self.include_changes_description = include_changes_description + self.include_failures = include_failures self.compress = compress self.buffer_size = buffer_size @@ -121,6 +124,11 @@ def __init__( # Create parent directory if needed self.output_path.parent.mkdir(parents=True, exist_ok=True) + # Failed iterations have no child program, so they go to their own JSONL file rather + # than into the trace, whose records (and exports) all describe a parent/child pair + stem = self.output_path.name.split(".")[0] + self.failures_path = self.output_path.with_name(f"{stem}_failures.jsonl") + # Initialize buffer for batched writing self.buffer: List[EvolutionTrace] = [] @@ -207,6 +215,43 @@ def log_trace( except Exception as e: logger.error(f"Error logging evolution trace: {e}") + def log_failure( + self, + iteration: int, + error: str, + parent_id: Optional[str] = None, + evolution_mode: Optional[str] = None, + prompt: Optional[Dict[str, str]] = None, + llm_response: Optional[str] = None, + token_usage: Optional[Dict[str, Any]] = None, + island_id: Optional[int] = None, + ) -> None: + """ + Record an iteration that failed before producing a child program + + Written straight to failures_path (no buffering) when include_failures is set. + The prompt and response are kept only when include_prompts is also set. + """ + if not self.enabled or not self.include_failures: + return + + record = { + "iteration": iteration, + "timestamp": time.time(), + "error": error, + "parent_id": parent_id, + "evolution_mode": evolution_mode, + "island_id": island_id, + "token_usage": token_usage, + } + if self.include_prompts: + record["prompt"] = prompt + record["llm_response"] = llm_response + try: + append_trace_jsonl(record, self.failures_path, compress=self.compress) + except Exception as e: + logger.error(f"Error logging failed iteration: {e}") + def _update_stats(self, trace: EvolutionTrace): """Update running statistics""" self.stats["total_traces"] += 1 diff --git a/openevolve/process_parallel.py b/openevolve/process_parallel.py index 9c590dcd30..0abf0fa2c8 100644 --- a/openevolve/process_parallel.py +++ b/openevolve/process_parallel.py @@ -6,6 +6,7 @@ import logging import multiprocessing as mp import pickle +import random import signal import time from concurrent.futures import Future, ProcessPoolExecutor @@ -74,6 +75,36 @@ class SerializableResult: error: Optional[str] = None target_island: Optional[int] = None # Island where child should be placed token_usage: Optional[Dict[str, Any]] = None + # "diff" or "full_rewrite": the mode this iteration prompted and parsed with + evolution_mode: Optional[str] = None + + +EVOLUTION_MODE_DIFF = "diff" +EVOLUTION_MODE_FULL_REWRITE = "full_rewrite" + + +def _choose_evolution_mode(config: Config, iteration: int) -> str: + """Pick diff or full rewrite for one iteration. + + With random_seed set, the choice is seeded from the seed and the iteration, so it does not + depend on which worker runs the iteration and does not disturb other random draws""" + if not config.diff_based_evolution: + return EVOLUTION_MODE_FULL_REWRITE + p = config.full_rewrite_probability + if p <= 0: + return EVOLUTION_MODE_DIFF + if p >= 1: + return EVOLUTION_MODE_FULL_REWRITE + if config.random_seed is not None: + rng = random.Random(f"{config.random_seed}:{iteration}:evolution_mode") + else: + rng = random.Random() + return EVOLUTION_MODE_FULL_REWRITE if rng.random() < p else EVOLUTION_MODE_DIFF + + +def _prompt_template_key(evolution_mode: Optional[str]) -> str: + """The user template an iteration in `evolution_mode` was prompted with""" + return "full_rewrite_user" if evolution_mode == EVOLUTION_MODE_FULL_REWRITE else "diff_user" def _worker_init(config_dict: dict, evaluation_file: str, parent_env: dict = None) -> None: @@ -182,6 +213,26 @@ def _run_iteration_worker( iteration: int, db_snapshot: Dict[str, Any], parent_id: str, inspiration_ids: List[str] ) -> SerializableResult: """Run a single iteration in a worker process""" + try: + evolution_mode = _choose_evolution_mode(_worker_config, iteration) + except Exception as e: + logger.exception(f"Error choosing evolution mode in worker iteration {iteration}") + return SerializableResult(error=str(e), iteration=iteration) + result = _run_iteration(iteration, db_snapshot, parent_id, inspiration_ids, evolution_mode) + # Recorded on failures too, so parse/compile failure rates can be compared per mode + result.evolution_mode = evolution_mode + return result + + +def _run_iteration( + iteration: int, + db_snapshot: Dict[str, Any], + parent_id: str, + inspiration_ids: List[str], + evolution_mode: str, +) -> SerializableResult: + """Run one iteration, prompting and parsing as `evolution_mode` says""" + diff_based = evolution_mode == EVOLUTION_MODE_DIFF try: # Lazy initialization _lazy_init_worker_components() @@ -235,7 +286,7 @@ def _run_iteration_worker( inspirations=[p.to_dict() for p in inspirations], language=_worker_config.language, evolution_round=iteration, - diff_based_evolution=_worker_config.diff_based_evolution, + diff_based_evolution=diff_based, program_artifacts=parent_artifacts, feature_dimensions=db_snapshot.get("feature_dimensions", []), current_changes_description=parent_changes_desc, @@ -263,8 +314,20 @@ def _run_iteration_worker( error="LLM returned None response", iteration=iteration, token_usage=token_usage ) + def _reject(error: str) -> SerializableResult: + """A failed iteration that still carries what the LLM was asked and answered""" + return SerializableResult( + error=error, + iteration=iteration, + token_usage=token_usage, + prompt=prompt, + llm_response=llm_response, + parent_id=parent.id, + target_island=db_snapshot.get("sampling_island"), + ) + # Parse response based on evolution mode - if _worker_config.diff_based_evolution: + if diff_based: from openevolve.utils.code_utils import ( apply_diff_blocks, extract_diffs, @@ -275,15 +338,9 @@ def _run_iteration_worker( try: diff_blocks = extract_diffs(llm_response, _worker_config.diff_pattern) except ValueError as exc: - return SerializableResult( - error=str(exc), iteration=iteration, token_usage=token_usage - ) + return _reject(error=str(exc)) if not diff_blocks: - return SerializableResult( - error="No valid diffs found in response", - iteration=iteration, - token_usage=token_usage, - ) + return _reject(error="No valid diffs found in response") if _worker_config.prompt.programs_as_changes_description: try: @@ -293,9 +350,7 @@ def _run_iteration_worker( changes_description_text=parent_changes_desc, ) except Exception as e: - return SerializableResult( - error=str(e), iteration=iteration, token_usage=token_usage - ) + return _reject(error=str(e)) child_code, _ = apply_diff_blocks(parent.code, code_blocks) child_changes_desc, desc_applied = apply_diff_blocks( @@ -308,10 +363,8 @@ def _run_iteration_worker( or not child_changes_desc.strip() or child_changes_desc.strip() == parent_changes_desc.strip() ): - return SerializableResult( - error="changes_description was not updated or empty, program is discarded", - iteration=iteration, - token_usage=token_usage, + return _reject( + error="changes_description was not updated or empty, program is discarded" ) changes_summary = format_diff_summary( @@ -323,13 +376,11 @@ def _run_iteration_worker( # All diffs applied only to code child_code, applied = apply_diff_blocks(parent.code, diff_blocks) if applied == 0: - return SerializableResult( + return _reject( error=( f"None of the {len(diff_blocks)} SEARCH block(s) matched the " "parent program" ), - iteration=iteration, - token_usage=token_usage, ) if applied < len(diff_blocks): logger.warning( @@ -337,11 +388,7 @@ def _run_iteration_worker( "SEARCH block(s) matched the parent program" ) if child_code == parent.code: - return SerializableResult( - error="Diff did not change the parent program", - iteration=iteration, - token_usage=token_usage, - ) + return _reject(error="Diff did not change the parent program") changes_summary = format_diff_summary( diff_blocks, max_line_len=_worker_config.prompt.diff_summary_max_line_len, @@ -352,18 +399,10 @@ def _run_iteration_worker( new_code = parse_full_rewrite(llm_response, _worker_config.language) if not new_code: - return SerializableResult( - error=f"No valid code found in response", - iteration=iteration, - token_usage=token_usage, - ) + return _reject(error=f"No valid code found in response") if new_code == parent.code: - return SerializableResult( - error="Rewrite is identical to the parent program", - iteration=iteration, - token_usage=token_usage, - ) + return _reject(error="Rewrite is identical to the parent program") child_code = new_code changes_summary = "Full rewrite" @@ -375,27 +414,19 @@ def _run_iteration_worker( try: enforced_code = enforce_evolve_blocks(parent.code, child_code) except ValueError as exc: - return SerializableResult( - error=str(exc), iteration=iteration, token_usage=token_usage - ) + return _reject(error=str(exc)) if enforced_code != child_code: logger.info( f"Iteration {iteration}: reverted edits outside the EVOLVE-BLOCK regions" ) child_code = enforced_code if child_code == parent.code: - return SerializableResult( - error="All edits were outside the EVOLVE-BLOCK regions", - iteration=iteration, - token_usage=token_usage, - ) + return _reject(error="All edits were outside the EVOLVE-BLOCK regions") # Check code length if len(child_code) > _worker_config.max_code_length: - return SerializableResult( - error=f"Generated code exceeds maximum length ({len(child_code)} > {_worker_config.max_code_length})", - iteration=iteration, - token_usage=token_usage, + return _reject( + error=f"Generated code exceeds maximum length ({len(child_code)} > {_worker_config.max_code_length})" ) # Evaluate the child program @@ -422,6 +453,7 @@ def _run_iteration_worker( "parent_metrics": parent.metrics, "island": parent_island, "token_usage": token_usage, + "evolution_mode": evolution_mode, }, ) @@ -803,9 +835,22 @@ async def run_evolution( if result.error: self.last_iteration_error = result.error + mode_str = f" [{result.evolution_mode}]" if result.evolution_mode else "" logger.warning( - f"Iteration {completed_iteration} error: {result.error}{token_str}" + f"Iteration {completed_iteration} error{mode_str}: " + f"{result.error}{token_str}" ) + if self.evolution_tracer: + self.evolution_tracer.log_failure( + iteration=completed_iteration, + error=result.error, + parent_id=result.parent_id, + evolution_mode=result.evolution_mode, + prompt=result.prompt, + llm_response=result.llm_response, + token_usage=result.token_usage, + island_id=result.target_island, + ) elif result.child_program_dict: successful_iterations += 1 @@ -851,17 +896,14 @@ async def run_evolution( "iteration_time": result.iteration_time, "changes": child_program.metadata.get("changes", ""), "token_usage": result.token_usage, + "evolution_mode": result.evolution_mode, }, ) # Log prompts if result.prompt: self.database.log_prompt( - template_key=( - "full_rewrite_user" - if not self.config.diff_based_evolution - else "diff_user" - ), + template_key=_prompt_template_key(result.evolution_mode), program_id=child_program.id, prompt=result.prompt, responses=[result.llm_response] if result.llm_response else [], diff --git a/tests/test_controller_trace_failures.py b/tests/test_controller_trace_failures.py new file mode 100644 index 0000000000..4695b56881 --- /dev/null +++ b/tests/test_controller_trace_failures.py @@ -0,0 +1,144 @@ +""" +Tests that, with evolution_trace.include_failures, the parallel controller writes +iterations that failed before producing a child to the failures file, with +their error, mode, prompt and LLM response +""" + +import asyncio +import json +import os +import shutil +import tempfile +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +os.environ.setdefault("OPENAI_API_KEY", "test") + +from openevolve.config import Config +from openevolve.database import Program, ProgramDatabase +from openevolve.evolution_trace import EvolutionTracer +from openevolve.process_parallel import ProcessParallelController, SerializableResult + +EVALUATOR_CODE = """ +def evaluate(program_path): + return {"combined_score": 0.5} +""" + + +def _future(result): + future = MagicMock() + future.done.return_value = True + future.result.return_value = result + return future + + +def _child(iteration, parent_id): + return SerializableResult( + child_program_dict={ + "id": f"child_{iteration}", + "code": f"def evolved_{iteration}(): return {iteration}", + "language": "python", + "parent_id": parent_id, + "generation": 1, + "metrics": {"combined_score": 0.7}, + "iteration_found": iteration, + "metadata": { + "changes": "test", + "island": 0, + "parent_metrics": {"combined_score": 0.25}, + "evolution_mode": "full_rewrite", + }, + }, + parent_id=parent_id, + iteration_time=0.1, + iteration=iteration, + target_island=0, + evolution_mode="full_rewrite", + ) + + +def _failure(iteration): + return SerializableResult( + error="No valid diffs found in response", + iteration=iteration, + parent_id="seed", + prompt={"system": "s", "user": "u"}, + llm_response="I would not change anything.", + token_usage={"completion_tokens": 7}, + target_island=0, + evolution_mode="diff", + ) + + +class TestControllerTraceFailures(unittest.TestCase): + def setUp(self): + self.test_dir = tempfile.mkdtemp() + self.eval_file = os.path.join(self.test_dir, "evaluator.py") + with open(self.eval_file, "w") as f: + f.write(EVALUATOR_CODE) + + self.config = Config() + self.config.evaluator.parallel_evaluations = 1 + self.config.evaluator.timeout = 10 + self.config.database.num_islands = 1 + self.config.database.in_memory = True + + self.database = ProgramDatabase(self.config.database) + self.database.add( + Program( + id="seed", + code="def seed(): return 0", + language="python", + metrics={"combined_score": 0.5}, + iteration_found=0, + ) + ) + self.trace_path = Path(self.test_dir) / "evolution_trace.jsonl" + self.tracer = EvolutionTracer( + output_path=str(self.trace_path), include_failures=True, buffer_size=1 + ) + + def tearDown(self): + shutil.rmtree(self.test_dir, ignore_errors=True) + + def _run(self, results): + async def run_test(): + controller = ProcessParallelController( + self.config, self.eval_file, self.database, evolution_tracer=self.tracer + ) + with patch.object( + controller, + "_submit_iteration", + side_effect=lambda i, island: _future(results[i]), + ): + controller.start() + try: + await controller.run_evolution( + start_iteration=1, max_iterations=len(results), target_score=None + ) + finally: + controller.stop() + + asyncio.run(run_test()) + self.tracer.close() + + def _read(self, path): + with open(path) as f: + return [json.loads(line) for line in f if line.strip()] + + def test_failed_iteration_goes_to_failures_file(self): + self._run({1: _failure(1), 2: _child(2, "seed")}) + + [failure] = self._read(self.tracer.failures_path) + self.assertEqual(failure["iteration"], 1) + self.assertEqual(failure["evolution_mode"], "diff") + self.assertEqual(failure["error"], "No valid diffs found in response") + self.assertEqual(failure["llm_response"], "I would not change anything.") + self.assertEqual(failure["parent_id"], "seed") + self.assertEqual(failure["island_id"], 0) + self.assertEqual([t["iteration"] for t in self._read(self.trace_path)], [2]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_diff_format_validation_worker.py b/tests/test_diff_format_validation_worker.py index 356943a038..0a8ee2c006 100644 --- a/tests/test_diff_format_validation_worker.py +++ b/tests/test_diff_format_validation_worker.py @@ -53,6 +53,7 @@ def setUp(self): self.sampler.build_prompt.return_value = {"system": "sys", "user": "usr"} def _run(self, llm_response): + self.llm_response = llm_response llm = MagicMock() llm.generate_with_context = AsyncMock(return_value=llm_response) llm.last_usage = USAGE @@ -72,6 +73,10 @@ def assert_rejected(self, result, message): self.assertIn(message, result.error) self.assertEqual(result.iteration, 3) self.assertEqual(result.token_usage, USAGE) + # Kept so failed iterations can be inspected (evolution_trace.include_failures) + self.assertEqual(result.llm_response, self.llm_response) + self.assertEqual(result.prompt, {"system": "sys", "user": "usr"}) + self.assertEqual(result.parent_id, "parent") self.evaluator.evaluate_program.assert_not_called() def evaluated_code(self): diff --git a/tests/test_evolution_trace.py b/tests/test_evolution_trace.py index a4c9d97c96..67b7202bab 100644 --- a/tests/test_evolution_trace.py +++ b/tests/test_evolution_trace.py @@ -650,5 +650,53 @@ def test_programs_dir_not_found(self): extract_evolution_trace_from_checkpoint(empty_checkpoint) +class TestTracerFailures(unittest.TestCase): + """Failed iterations go to a separate file, only when include_failures is set""" + + FAILURE = dict( + iteration=4, + error="No valid diffs found in response", + parent_id="p1", + evolution_mode="diff", + prompt={"system": "s", "user": "u"}, + llm_response="no diff here", + token_usage={"completion_tokens": 3}, + island_id=1, + ) + + def _tracer(self, tmp, **kwargs): + return EvolutionTracer(output_path=str(Path(tmp) / "trace.jsonl"), **kwargs) + + def _read(self, path): + with open(path) as f: + return [json.loads(line) for line in f if line.strip()] + + def test_writes_failure_record(self): + with tempfile.TemporaryDirectory() as tmp: + tracer = self._tracer(tmp, include_failures=True) + tracer.log_failure(**self.FAILURE) + self.assertEqual(tracer.failures_path, Path(tmp) / "trace_failures.jsonl") + [record] = self._read(tracer.failures_path) + for key, value in self.FAILURE.items(): + self.assertEqual(record[key], value) + # The trace itself only ever holds parent/child records + self.assertFalse((Path(tmp) / "trace.jsonl").exists()) + + def test_prompt_and_response_follow_include_prompts(self): + with tempfile.TemporaryDirectory() as tmp: + tracer = self._tracer(tmp, include_failures=True, include_prompts=False) + tracer.log_failure(**self.FAILURE) + [record] = self._read(tracer.failures_path) + self.assertNotIn("llm_response", record) + self.assertNotIn("prompt", record) + self.assertEqual(record["error"], self.FAILURE["error"]) + + def test_off_by_default(self): + with tempfile.TemporaryDirectory() as tmp: + tracer = self._tracer(tmp) + tracer.log_failure(**self.FAILURE) + self.assertFalse(tracer.failures_path.exists()) + + if __name__ == "__main__": unittest.main() \ No newline at end of file diff --git a/tests/test_full_rewrite_probability.py b/tests/test_full_rewrite_probability.py new file mode 100644 index 0000000000..a6b819eb98 --- /dev/null +++ b/tests/test_full_rewrite_probability.py @@ -0,0 +1,214 @@ +""" +Tests for mixing diff-based iterations with full rewrites via +full_rewrite_probability (issue #116) +""" + +import os +import unittest +from unittest.mock import AsyncMock, MagicMock, patch + +os.environ.setdefault("OPENAI_API_KEY", "test") + +from openevolve import process_parallel as process_parallel_module +from openevolve.config import Config +from openevolve.database import Program +from openevolve.process_parallel import ( + EVOLUTION_MODE_DIFF, + EVOLUTION_MODE_FULL_REWRITE, + _choose_evolution_mode, + _prompt_template_key, +) + +PARENT_CODE = "x = 1\ny = 1" +DIFF_RESPONSE = "<<<<<<< SEARCH\nx = 1\n=======\nx = 2\n>>>>>>> REPLACE\n" +REWRITE_RESPONSE = "```python\nx = 3\ny = 3\n```" + + +def _config(**kwargs): + config = Config() + config.language = "python" + for key, value in kwargs.items(): + setattr(config, key, value) + return config + + +class TestConfigValidation(unittest.TestCase): + def _from_dict(self, **kwargs): + return Config.from_dict(kwargs) + + def test_default_is_zero(self): + self.assertEqual(Config().full_rewrite_probability, 0.0) + self.assertEqual(self._from_dict().full_rewrite_probability, 0.0) + + def test_accepts_bounds(self): + for p in (0, 0.0, 0.2, 1, 1.0): + self.assertEqual( + self._from_dict(full_rewrite_probability=p).full_rewrite_probability, p + ) + + def test_rejects_out_of_range(self): + for p in (-0.1, 1.5, float("nan"), float("inf")): + with self.assertRaisesRegex(ValueError, "between 0 and 1"): + self._from_dict(full_rewrite_probability=p) + + def test_rejects_with_diff_based_evolution_off(self): + with self.assertRaisesRegex(ValueError, "only applies when diff_based_evolution=true"): + self._from_dict(diff_based_evolution=False, full_rewrite_probability=0.2) + # Zero is fine: it is the default + self._from_dict(diff_based_evolution=False, full_rewrite_probability=0.0) + + def test_rejects_with_changes_description(self): + with self.assertRaisesRegex(ValueError, "full_rewrite_probability=0"): + self._from_dict( + full_rewrite_probability=0.2, + prompt={"programs_as_changes_description": True}, + ) + self._from_dict(prompt={"programs_as_changes_description": True}) + + def test_existing_changes_description_check_still_applies(self): + with self.assertRaisesRegex(ValueError, "requires diff_based_evolution=true"): + self._from_dict( + diff_based_evolution=False, prompt={"programs_as_changes_description": True} + ) + + def test_config_built_in_code_is_validated_by_controller(self): + from openevolve.controller import OpenEvolve + + config = _config(full_rewrite_probability=2.0) + with self.assertRaisesRegex(ValueError, "between 0 and 1"): + OpenEvolve("initial_program.py", "evaluator.py", config=config) + + +class TestChooseEvolutionMode(unittest.TestCase): + def test_fixed_modes(self): + self.assertEqual( + _choose_evolution_mode(_config(full_rewrite_probability=0.0), 1), EVOLUTION_MODE_DIFF + ) + self.assertEqual( + _choose_evolution_mode(_config(full_rewrite_probability=1.0), 1), + EVOLUTION_MODE_FULL_REWRITE, + ) + self.assertEqual( + _choose_evolution_mode(_config(diff_based_evolution=False), 1), + EVOLUTION_MODE_FULL_REWRITE, + ) + + def test_seeded_choice_depends_on_iteration_only(self): + config = _config(full_rewrite_probability=0.3, random_seed=7) + first = [_choose_evolution_mode(config, i) for i in range(200)] + # Same iterations in reverse order give the same modes + second = [_choose_evolution_mode(config, i) for i in reversed(range(200))][::-1] + self.assertEqual(first, second) + + def test_does_not_touch_global_random_state(self): + import random + + config = _config(full_rewrite_probability=0.5, random_seed=3) + state = random.getstate() + for i in range(10): + _choose_evolution_mode(config, i) + self.assertEqual(random.getstate(), state) + + def test_template_key_matches_mode(self): + self.assertEqual(_prompt_template_key(EVOLUTION_MODE_DIFF), "diff_user") + self.assertEqual(_prompt_template_key(EVOLUTION_MODE_FULL_REWRITE), "full_rewrite_user") + + def test_different_seeds_differ(self): + a = [ + _choose_evolution_mode(_config(full_rewrite_probability=0.5, random_seed=1), i) + for i in range(64) + ] + b = [ + _choose_evolution_mode(_config(full_rewrite_probability=0.5, random_seed=2), i) + for i in range(64) + ] + self.assertNotEqual(a, b) + + def test_rate_roughly_matches_probability(self): + config = _config(full_rewrite_probability=0.2, random_seed=42) + modes = [_choose_evolution_mode(config, i) for i in range(5000)] + rate = modes.count(EVOLUTION_MODE_FULL_REWRITE) / len(modes) + self.assertAlmostEqual(rate, 0.2, delta=0.03) + + def test_unseeded_still_mixes(self): + config = _config(full_rewrite_probability=0.5, random_seed=None) + modes = {_choose_evolution_mode(config, i) for i in range(200)} + self.assertEqual(modes, {EVOLUTION_MODE_DIFF, EVOLUTION_MODE_FULL_REWRITE}) + + +class TestWorkerUsesChosenMode(unittest.TestCase): + def setUp(self): + parent = Program( + id="parent", code=PARENT_CODE, metrics={"combined_score": 0.5}, metadata={"island": 0} + ) + self.snapshot = { + "programs": {"parent": parent.to_dict()}, + "artifacts": {}, + "islands": [["parent"]], + "current_island": 0, + "sampling_island": 0, + "feature_dimensions": [], + } + self.evaluator = MagicMock() + self.evaluator.evaluate_program = AsyncMock(return_value={"combined_score": 1.0}) + self.evaluator.get_pending_artifacts.return_value = None + self.sampler = MagicMock() + self.sampler.build_prompt.return_value = {"system": "sys", "user": "usr"} + + def _run(self, config, llm_response): + llm = MagicMock() + llm.generate_with_context = AsyncMock(return_value=llm_response) + llm.last_usage = None + with patch.multiple( + process_parallel_module, + create=True, + _worker_config=config, + _worker_llm_ensemble=llm, + _worker_prompt_sampler=self.sampler, + _worker_evaluator=self.evaluator, + ): + return process_parallel_module._run_iteration_worker(3, self.snapshot, "parent", []) + + def _prompted_diff_based(self): + return self.sampler.build_prompt.call_args.kwargs["diff_based_evolution"] + + def test_full_rewrite_iteration(self): + result = self._run(_config(full_rewrite_probability=1.0), REWRITE_RESPONSE) + self.assertIsNone(result.error) + self.assertFalse(self._prompted_diff_based()) + self.assertEqual(result.evolution_mode, EVOLUTION_MODE_FULL_REWRITE) + child = result.child_program_dict + self.assertEqual(child["code"], "x = 3\ny = 3") + self.assertEqual(child["metadata"]["evolution_mode"], EVOLUTION_MODE_FULL_REWRITE) + + def test_diff_iteration(self): + result = self._run(_config(full_rewrite_probability=0.0), DIFF_RESPONSE) + self.assertIsNone(result.error) + self.assertTrue(self._prompted_diff_based()) + self.assertEqual(result.evolution_mode, EVOLUTION_MODE_DIFF) + self.assertEqual( + result.child_program_dict["metadata"]["evolution_mode"], EVOLUTION_MODE_DIFF + ) + + def test_mode_choice_failure_is_returned_not_raised(self): + with patch.object( + process_parallel_module, "_choose_evolution_mode", side_effect=RuntimeError("boom") + ): + result = self._run(_config(full_rewrite_probability=0.5), DIFF_RESPONSE) + self.assertEqual(result.error, "boom") + self.assertIsNone(result.evolution_mode) + self.evaluator.evaluate_program.assert_not_called() + + def test_mode_recorded_on_failure(self): + # An empty response is rejected by the rewrite parser + result = self._run(_config(full_rewrite_probability=1.0), "") + self.assertIsNotNone(result.error) + self.assertEqual(result.evolution_mode, EVOLUTION_MODE_FULL_REWRITE) + + result = self._run(_config(full_rewrite_probability=0.0), "no diffs here") + self.assertIn("No valid diffs", result.error) + self.assertEqual(result.evolution_mode, EVOLUTION_MODE_DIFF) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_worker_config_serialization.py b/tests/test_worker_config_serialization.py index 32ae5930e3..b88fa091d9 100644 --- a/tests/test_worker_config_serialization.py +++ b/tests/test_worker_config_serialization.py @@ -47,6 +47,7 @@ def setUp(self): self.config.language = "rust" self.config.file_suffix = ".rs" self.config.diff_based_evolution = False + self.config.full_rewrite_probability = 0.25 self.config.max_code_length = 1234 self.config.enforce_evolve_blocks = True self.config.diff_pattern = CUSTOM_DIFF_PATTERN From b1120191e61f9f41eba88c3252331a7a909b9585 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 13:10:47 +0800 Subject: [PATCH 2/5] Round-trip list metadata through HDF5 trace exports The exporter JSON-encodes non-scalar metadata but the loader only decoded strings starting with "{", so lists came back as strings. Decoding every "["-prefixed string instead would corrupt real strings that look like JSON. The exporter now records which attributes it encoded in __json_encoded_keys__, and the loader decodes exactly those. Files without the list load as before, since their values can't be told apart. Closes #574 Co-Authored-By: Claude Opus 5.5 --- openevolve/utils/trace_export_utils.py | 19 ++++++- tests/test_trace_export_hdf5.py | 69 ++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 tests/test_trace_export_hdf5.py diff --git a/openevolve/utils/trace_export_utils.py b/openevolve/utils/trace_export_utils.py index 218ce4323d..967f82d71f 100644 --- a/openevolve/utils/trace_export_utils.py +++ b/openevolve/utils/trace_export_utils.py @@ -80,6 +80,11 @@ def export_traces_json( logger.info(f"Exported {len(traces)} traces to {output_path}") +# Names the metadata attributes that were JSON-encoded, so a list or dict can be told apart +# from a string that merely looks like JSON +_HDF5_JSON_KEYS_ATTR = "__json_encoded_keys__" + + def export_traces_hdf5( traces: List[Any], output_path: Union[str, Path], metadata: Optional[Dict[str, Any]] = None ) -> None: @@ -107,12 +112,15 @@ def export_traces_hdf5( meta_group = f.create_group("metadata") # Add metadata + json_encoded_keys = [] if metadata: for key, value in metadata.items(): if isinstance(value, (str, int, float, bool)): meta_group.attrs[key] = value else: meta_group.attrs[key] = json.dumps(value) + json_encoded_keys.append(key) + meta_group.attrs[_HDF5_JSON_KEYS_ATTR] = json.dumps(json_encoded_keys) meta_group.attrs["total_traces"] = len(traces) meta_group.attrs["exported_at"] = time.time() @@ -252,10 +260,17 @@ def load_traces_hdf5(input_path: Union[str, Path]) -> tuple[List[Dict[str, Any]] # Load metadata if "metadata" in f: meta_group = f["metadata"] + json_encoded_keys = meta_group.attrs.get(_HDF5_JSON_KEYS_ATTR) + if json_encoded_keys is not None: + json_encoded_keys = set(json.loads(json_encoded_keys)) for key in meta_group.attrs: + if key == _HDF5_JSON_KEYS_ATTR: + continue value = meta_group.attrs[key] - # Try to parse JSON strings - if isinstance(value, str) and value.startswith("{"): + if json_encoded_keys is not None: + metadata[key] = json.loads(value) if key in json_encoded_keys else value + # Files without the key list can't tell JSON from a string that looks like it + elif isinstance(value, str) and value.startswith("{"): try: metadata[key] = json.loads(value) except json.JSONDecodeError: diff --git a/tests/test_trace_export_hdf5.py b/tests/test_trace_export_hdf5.py new file mode 100644 index 0000000000..93c55f83f5 --- /dev/null +++ b/tests/test_trace_export_hdf5.py @@ -0,0 +1,69 @@ +"""Regression tests for HDF5 trace metadata round trips (#574).""" + +import tempfile +import unittest +from pathlib import Path + +from openevolve.utils.trace_export_utils import export_traces, load_traces + +try: + import h5py +except ImportError: + h5py = None + + +@unittest.skipIf(h5py is None, "h5py is not installed") +class TestHdf5MetadataRoundTrip(unittest.TestCase): + def _round_trip(self, metadata): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "trace.h5" + export_traces([{"iteration": 1}], path, format="hdf5", metadata=metadata) + _, loaded = load_traces(path) + return loaded + + def test_lists_stay_lists(self): + metadata = {"labels": ["alpha"], "empty": [], "nested": [{"a": [1, 2]}]} + loaded = self._round_trip(metadata) + for key, value in metadata.items(): + self.assertEqual(loaded[key], value) + + def test_dicts_and_none_round_trip(self): + loaded = self._round_trip({"config": {"a": 1}, "nothing": None}) + self.assertEqual(loaded["config"], {"a": 1}) + self.assertIsNone(loaded["nothing"]) + + def test_json_looking_strings_stay_strings(self): + metadata = {"list": ["alpha"], "literal": '["alpha"]', "object": '{"a": 1}', "bad": "[x"} + loaded = self._round_trip(metadata) + self.assertEqual(loaded["list"], ["alpha"]) + self.assertEqual(loaded["literal"], '["alpha"]') + self.assertEqual(loaded["object"], '{"a": 1}') + self.assertEqual(loaded["bad"], "[x") + + def test_scalars_and_generated_fields_are_unchanged(self): + loaded = self._round_trip({"name": "run", "count": 3, "rate": 0.5, "flag": True}) + self.assertEqual(loaded["name"], "run") + self.assertEqual(loaded["count"], 3) + self.assertEqual(loaded["rate"], 0.5) + self.assertTrue(loaded["flag"]) + self.assertEqual(loaded["total_traces"], 1) + self.assertIn("exported_at", loaded) + self.assertNotIn("__json_encoded_keys__", loaded) + + def test_files_without_a_key_list_load_as_before(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "legacy.h5" + with h5py.File(path, "w") as f: + f.create_group("traces") + attrs = f.create_group("metadata").attrs + attrs["config"] = '{"a": 1}' + attrs["labels"] = '["alpha"]' + attrs["broken"] = "{not json" + _, loaded = load_traces(path) + self.assertEqual(loaded["config"], {"a": 1}) + self.assertEqual(loaded["labels"], '["alpha"]') + self.assertEqual(loaded["broken"], "{not json") + + +if __name__ == "__main__": + unittest.main() From db31027b2b5c4782ae9276ee505bcf1b930b198f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 13:10:47 +0800 Subject: [PATCH 3/5] Complete failure tracing and record each program's LLM model Review fixes for evolution_trace.include_failures: - Timed-out and crashed iterations now reach the failures file; they were the failures with the least other evidence and were missing from it. - Failed LLM calls keep the prompt, parent and island like later failures. - failures_path now matches the file written when compress is on (it lacked .gz). Dotted trace names no longer share one failures file: run.v2 and run.v3 both wrote to run_failures.jsonl. Each child now stores the sampled model as metadata["llm_model"], also in the trace, so ensemble models can be compared. token_usage only names the model when the provider returns usage data. Closes #381 Co-Authored-By: Claude Opus 5.5 --- configs/default_config.yaml | 2 +- openevolve/evolution_trace.py | 8 ++-- openevolve/llm/ensemble.py | 5 ++- openevolve/process_parallel.py | 42 +++++++++++++-------- tests/test_controller_trace_failures.py | 27 ++++++++++++- tests/test_diff_format_validation_worker.py | 39 ++++++++++++++++++- tests/test_evolution_trace.py | 34 +++++++++++++++++ tests/test_llm_ensemble.py | 9 +++++ 8 files changed, 142 insertions(+), 24 deletions(-) diff --git a/configs/default_config.yaml b/configs/default_config.yaml index 1793b90354..9f7ed90b6e 100644 --- a/configs/default_config.yaml +++ b/configs/default_config.yaml @@ -185,7 +185,7 @@ evolution_trace: include_prompts: true # Include prompts and LLM responses include_failures: false # Also log failed iterations (error, mode, and with # include_prompts the prompt and response) to - # _failures.jsonl + # _failures.jsonl (.jsonl.gz with compress) output_path: null # Path for trace output (defaults to output_dir/evolution_trace.{format}) buffer_size: 10 # Number of traces to buffer before writing compress: false # Compress output file (jsonl only) diff --git a/openevolve/evolution_trace.py b/openevolve/evolution_trace.py index 8e0e78d7e5..f899e8d51c 100644 --- a/openevolve/evolution_trace.py +++ b/openevolve/evolution_trace.py @@ -127,8 +127,9 @@ def __init__( # Failed iterations have no child program, so they go to their own JSONL file rather # than into the trace, whose records (and exports) all describe a parent/child pair - stem = self.output_path.name.split(".")[0] - self.failures_path = self.output_path.with_name(f"{stem}_failures.jsonl") + name = self.output_path.name.removesuffix(".gz") + suffix = ".jsonl.gz" if self.compress else ".jsonl" + self.failures_path = self.output_path.with_name(f"{Path(name).stem}_failures{suffix}") # Initialize buffer for batched writing self.buffer: List[EvolutionTrace] = [] @@ -232,7 +233,8 @@ def log_failure( island_id: Optional[int] = None, ) -> None: """ - Record an iteration that failed before producing a child program + Record an iteration that did not yield a child program, including ones that timed + out or whose worker crashed (those have no prompt, response or mode to report) Written straight to failures_path (no buffering) when include_failures is set. The prompt and response are kept only when include_prompts is also set. diff --git a/openevolve/llm/ensemble.py b/openevolve/llm/ensemble.py index 402afc8884..72fa82fe8c 100644 --- a/openevolve/llm/ensemble.py +++ b/openevolve/llm/ensemble.py @@ -98,6 +98,8 @@ def __init__(self, models_cfg: List[LLMModelConfig]): logger._ensemble_logged = True self.last_usage: Optional[Dict[str, Any]] = None + # Configured name of the model sampled for the latest request + self.last_model: Optional[str] = None # Set by reseed(); gives each iteration's requests their own seed self._iteration: Optional[int] = None @@ -144,7 +146,8 @@ def _request_kwargs(self, index: int, kwargs: Dict[str, Any]) -> Dict[str, Any]: def _sample_index(self) -> int: """Index of a model sampled from the ensemble based on weights""" index = self.random_state.choices(range(len(self.models)), weights=self.weights, k=1)[0] - logger.info(f"Sampled model: {self.models_cfg[index].name}") + self.last_model = self.models_cfg[index].name + logger.info(f"Sampled model: {self.last_model}") return index def _sample_model(self) -> LLMInterface: diff --git a/openevolve/process_parallel.py b/openevolve/process_parallel.py index 00d55e92b6..537d03723a 100644 --- a/openevolve/process_parallel.py +++ b/openevolve/process_parallel.py @@ -295,6 +295,20 @@ def _run_iteration( ) iteration_start = time.time() + llm_response = None + token_usage = None + + def _reject(error: str) -> SerializableResult: + """A failed iteration that still carries what the LLM was asked and answered""" + return SerializableResult( + error=error, + iteration=iteration, + token_usage=token_usage, + prompt=prompt, + llm_response=llm_response, + parent_id=parent.id, + target_island=db_snapshot.get("sampling_island"), + ) # Generate code modification (sync wrapper for async) try: @@ -307,25 +321,11 @@ def _run_iteration( token_usage = getattr(_worker_llm_ensemble, "last_usage", None) except Exception as e: logger.error(f"LLM generation failed: {e}") - return SerializableResult(error=f"LLM generation failed: {str(e)}", iteration=iteration) + return _reject(error=f"LLM generation failed: {str(e)}") # Check for None response if llm_response is None: - return SerializableResult( - error="LLM returned None response", iteration=iteration, token_usage=token_usage - ) - - def _reject(error: str) -> SerializableResult: - """A failed iteration that still carries what the LLM was asked and answered""" - return SerializableResult( - error=error, - iteration=iteration, - token_usage=token_usage, - prompt=prompt, - llm_response=llm_response, - parent_id=parent.id, - target_island=db_snapshot.get("sampling_island"), - ) + return _reject(error="LLM returned None response") # Parse response based on evolution mode if diff_based: @@ -455,6 +455,7 @@ def _reject(error: str) -> SerializableResult: "island": parent_island, "token_usage": token_usage, "evolution_mode": evolution_mode, + "llm_model": getattr(_worker_llm_ensemble, "last_model", None), }, ) @@ -917,6 +918,7 @@ async def run_evolution( "changes": child_program.metadata.get("changes", ""), "token_usage": result.token_usage, "evolution_mode": result.evolution_mode, + "llm_model": child_program.metadata.get("llm_model"), }, ) @@ -1100,11 +1102,19 @@ async def run_evolution( ) # Cancel the future to clean up the process future.cancel() + if self.evolution_tracer: + self.evolution_tracer.log_failure( + iteration=completed_iteration, error=self.last_iteration_error + ) except Exception as e: if getattr(e, "_openevolve_strategy_error", False): raise self.last_iteration_error = str(e) logger.error(f"Error processing result from iteration {completed_iteration}: {e}") + if self.evolution_tracer: + self.evolution_tracer.log_failure( + iteration=completed_iteration, error=self.last_iteration_error + ) completed_iterations += 1 diff --git a/tests/test_controller_trace_failures.py b/tests/test_controller_trace_failures.py index 4695b56881..c271ca110b 100644 --- a/tests/test_controller_trace_failures.py +++ b/tests/test_controller_trace_failures.py @@ -10,6 +10,7 @@ import shutil import tempfile import unittest +from concurrent.futures import TimeoutError as FutureTimeoutError from pathlib import Path from unittest.mock import MagicMock, patch @@ -29,7 +30,10 @@ def evaluate(program_path): def _future(result): future = MagicMock() future.done.return_value = True - future.result.return_value = result + if isinstance(result, BaseException): + future.result.side_effect = result + else: + future.result.return_value = result return future @@ -139,6 +143,27 @@ def test_failed_iteration_goes_to_failures_file(self): self.assertEqual(failure["island_id"], 0) self.assertEqual([t["iteration"] for t in self._read(self.trace_path)], [2]) + def test_timed_out_iteration_goes_to_failures_file(self): + self._run({1: FutureTimeoutError(), 2: _child(2, "seed")}) + + [failure] = self._read(self.tracer.failures_path) + self.assertEqual(failure["iteration"], 1) + self.assertIn("timed out", failure["error"]) + self.assertIsNone(failure["evolution_mode"]) + + def test_crashed_iteration_goes_to_failures_file(self): + self._run({1: RuntimeError("process pool broke"), 2: _child(2, "seed")}) + + [failure] = self._read(self.tracer.failures_path) + self.assertEqual(failure["iteration"], 1) + self.assertEqual(failure["error"], "process pool broke") + + def test_nothing_recorded_without_include_failures(self): + self.tracer.include_failures = False + self._run({1: FutureTimeoutError(), 2: RuntimeError("boom"), 3: _failure(3)}) + + self.assertFalse(self.tracer.failures_path.exists()) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_diff_format_validation_worker.py b/tests/test_diff_format_validation_worker.py index 0a8ee2c006..22e5b7960f 100644 --- a/tests/test_diff_format_validation_worker.py +++ b/tests/test_diff_format_validation_worker.py @@ -18,6 +18,7 @@ VALID_BLOCK = "<<<<<<< SEARCH\nx = 1\n=======\nx = 2\n>>>>>>> REPLACE\n" EXTRA_SEPARATOR = "<<<<<<< SEARCH\nx = 1\n=======\nx = 2\n=======\n>>>>>>> REPLACE\n" USAGE = {"prompt_tokens": 11, "completion_tokens": 7, "total_tokens": 18} +MODEL = "test-model" class _WorkerTestBase(unittest.TestCase): @@ -52,11 +53,12 @@ def setUp(self): self.sampler = MagicMock() self.sampler.build_prompt.return_value = {"system": "sys", "user": "usr"} - def _run(self, llm_response): + def _run(self, llm_response, llm_error=None): self.llm_response = llm_response llm = MagicMock() - llm.generate_with_context = AsyncMock(return_value=llm_response) + llm.generate_with_context = AsyncMock(return_value=llm_response, side_effect=llm_error) llm.last_usage = USAGE + llm.last_model = MODEL # Worker globals are only created in worker processes, hence create=True. with patch.multiple( process_parallel_module, @@ -129,6 +131,39 @@ def test_custom_diff_pattern_keeps_permissive_parsing(self): self.evaluator.evaluate_program.assert_called_once() +class TestWorkerLLMFailures(_WorkerTestBase): + """Failed LLM calls still say what was asked, for evolution_trace.include_failures""" + + def _assert_keeps_context(self, result, message): + self.assertIsNone(result.child_program_dict) + self.assertIn(message, result.error) + self.assertEqual(result.prompt, {"system": "sys", "user": "usr"}) + self.assertEqual(result.parent_id, "parent") + self.assertEqual(result.target_island, 0) + self.assertEqual(result.evolution_mode, "diff") + self.assertIsNone(result.llm_response) + self.evaluator.evaluate_program.assert_not_called() + + def test_llm_exception(self): + result = self._run(None, llm_error=RuntimeError("connection refused")) + self._assert_keeps_context(result, "LLM generation failed: connection refused") + self.assertIsNone(result.token_usage) + + def test_llm_returns_none(self): + result = self._run(None) + self._assert_keeps_context(result, "LLM returned None response") + self.assertEqual(result.token_usage, USAGE) + + +class TestWorkerRecordsModel(_WorkerTestBase): + """Each program records the model that generated it (#381)""" + + def test_child_metadata_has_llm_model(self): + result = self._run(VALID_BLOCK) + self.assertIsNone(result.error) + self.assertEqual(result.child_program_dict["metadata"]["llm_model"], MODEL) + + class TestWorkerUnappliedDiffs(_WorkerTestBase): """Diffs that do not change the parent are reported, not evaluated (#346)""" diff --git a/tests/test_evolution_trace.py b/tests/test_evolution_trace.py index 67b7202bab..3da2222658 100644 --- a/tests/test_evolution_trace.py +++ b/tests/test_evolution_trace.py @@ -2,6 +2,7 @@ Tests for evolution trace functionality """ +import gzip import json import os import tempfile @@ -650,6 +651,39 @@ def test_programs_dir_not_found(self): extract_evolution_trace_from_checkpoint(empty_checkpoint) +class TestTracerFailuresPath(unittest.TestCase): + """The failures file is named after the whole trace name, and exists where it says""" + + def _path(self, tmp, name, **kwargs): + tracer = EvolutionTracer(output_path=str(Path(tmp) / name), **kwargs) + tracer.close() + return tracer.failures_path + + def test_dotted_trace_names_do_not_share_a_failures_file(self): + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual(self._path(tmp, "run.v2.jsonl").name, "run.v2_failures.jsonl") + self.assertEqual(self._path(tmp, "run.v3.jsonl").name, "run.v3_failures.jsonl") + + def test_hidden_trace_name(self): + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual(self._path(tmp, ".trace.jsonl").name, ".trace_failures.jsonl") + + def test_compressed_failures_are_written_to_failures_path(self): + with tempfile.TemporaryDirectory() as tmp: + tracer = EvolutionTracer( + output_path=str(Path(tmp) / "trace.jsonl"), compress=True, include_failures=True + ) + tracer.log_failure(iteration=1, error="boom") + tracer.close() + + self.assertEqual(tracer.failures_path.name, "trace_failures.jsonl.gz") + self.assertEqual( + [p.name for p in Path(tmp).glob("*failures*")], ["trace_failures.jsonl.gz"] + ) + with gzip.open(tracer.failures_path, "rt") as f: + self.assertEqual(json.loads(f.readline())["error"], "boom") + + class TestTracerFailures(unittest.TestCase): """Failed iterations go to a separate file, only when include_failures is set""" diff --git a/tests/test_llm_ensemble.py b/tests/test_llm_ensemble.py index 189a117133..e324aae9ab 100644 --- a/tests/test_llm_ensemble.py +++ b/tests/test_llm_ensemble.py @@ -64,6 +64,14 @@ def test_ensemble_initialization(self): self.assertEqual(ensemble.models[1].model, "b") self.assertEqual(ensemble.models[1].some_field, "value") + def test_last_model_tracks_the_sampled_model(self): + models = [LLMModelConfig(name="a", weight=1.0), LLMModelConfig(name="b", weight=1.0)] + ensemble = LLMEnsemble(models) + self.assertIsNone(ensemble.last_model) + for _ in range(20): + index = ensemble._sample_index() + self.assertEqual(ensemble.last_model, models[index].name) + class TestTokenUsageLogging(unittest.IsolatedAsyncioTestCase): async def test_openai_llm_token_usage_logging_and_propagation(self): @@ -116,6 +124,7 @@ async def test_openai_llm_token_usage_logging_and_propagation(self): messages=[{"role": "user", "content": "user"}], ) self.assertEqual(ensemble.last_usage, llm.last_usage) + self.assertEqual(ensemble.last_model, cfg.name) class TestCustomClientSampling(unittest.IsolatedAsyncioTestCase): From ffc173830c074e9fdedc98e9415b806f14c081a4 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 13:10:47 +0800 Subject: [PATCH 4/5] Bump version to 0.4.1 Co-Authored-By: Claude Opus 5.5 --- openevolve/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openevolve/_version.py b/openevolve/_version.py index 350f9834a1..335136d1aa 100644 --- a/openevolve/_version.py +++ b/openevolve/_version.py @@ -1,3 +1,3 @@ """Version information for openevolve package.""" -__version__ = "0.4.0" +__version__ = "0.4.1" From eda9f5c92317c3ba7d9387fb17ca1fc4af56a471 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 13:35:05 +0800 Subject: [PATCH 5/5] Streamline the README and lead with research adoption Cut the README from 1,074 to about 200 lines. It now opens with adoption and a list of papers using OpenEvolve, then results, quick start and an overview. Emojis are gone, which also fixes the Quick Start and Installation anchors they were breaking, and the dead example output link is removed. The quick start now clones the repo, since the examples are not part of the pip package. Detailed material moves to docs/: LLM providers, configuration, system messages, showcase, how it works, FAQ and roadmap. Co-Authored-By: Claude Opus 5.5 --- README.md | 1073 ++++----------------------------------- docs/configuration.md | 257 ++++++++++ docs/faq.md | 55 ++ docs/how-it-works.md | 35 ++ docs/llm-providers.md | 144 ++++++ docs/roadmap.md | 17 + docs/showcase.md | 88 ++++ docs/system-messages.md | 151 ++++++ 8 files changed, 847 insertions(+), 973 deletions(-) create mode 100644 docs/configuration.md create mode 100644 docs/faq.md create mode 100644 docs/how-it-works.md create mode 100644 docs/llm-providers.md create mode 100644 docs/roadmap.md create mode 100644 docs/showcase.md create mode 100644 docs/system-messages.md diff --git a/README.md b/README.md index 04ccc5fca4..783c9e376c 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ OpenEvolve Logo -**🧬 The most advanced open-source evolutionary coding agent** +**The most advanced open-source evolutionary coding agent** *Turn your LLMs into autonomous code optimizers that discover breakthrough algorithms* @@ -14,1040 +14,178 @@ License

-[🚀 **Quick Start**](#quick-start) • [**Examples**](#examples-gallery) • [**System Messages**](#crafting-effective-system-messages) • [**Discussions**](https://github.com/algorithmicsuperintelligence/openevolve/discussions) - -*From random search to state-of-the-art: Watch your code evolve in real-time* +[Quick Start](#quick-start) • [Used In Research](#used-in-research) • [Examples](#examples) • [Docs](#documentation) • [Discussions](https://github.com/algorithmicsuperintelligence/openevolve/discussions) --- -## Why OpenEvolve? +OpenEvolve is an open-source implementation of Google DeepMind's AlphaEvolve. It uses LLMs to mutate programs and evolves them with MAP-Elites and island populations, so you can point it at any problem you can score. It has become a standard tool and baseline for LLM-driven algorithm discovery: **cited in hundreds of research papers**, with 7,600+ GitHub stars, 1,200+ forks, 80+ contributors and 17,000+ monthly PyPI downloads. - - - - - - -
+## Used In Research -### **Autonomous Discovery** -LLMs don't just optimize—they **discover** entirely new algorithms. No human guidance needed. +**Producing new results** - +| Paper | How OpenEvolve is used | +|-------|------------------------| +| [New Bounds for Zarankiewicz Numbers via Reinforced LLM Evolutionary Search](https://arxiv.org/abs/2605.01120) | New bounds in extremal combinatorics, obtained with OpenEvolve | +| [Let the Barbarians In: How AI Can Accelerate Systems Performance Research](https://arxiv.org/abs/2512.14806) | Ten systems performance case studies, using OpenEvolve among other tools | +| [Algorithmic Algorithm Development with LLMs](https://arxiv.org/abs/2606.01975) | Contraction order optimization for tensor networks | +| [Even with AI, Bijection Discovery is Still Hard](https://arxiv.org/abs/2511.20987) | Combinatorial bijection discovery | -### **Proven Results** -**2-3x speedups** on real hardware. **State-of-the-art** circle packing. **Breakthrough** optimizations. +**Security research** - +| Paper | How OpenEvolve is used | +|-------|------------------------| +| [Trojan Hippo Bench](https://arxiv.org/abs/2605.01970) | Adaptive red-teaming benchmark for persistent memory attacks on LLM agents | +| [Divide and Inject](https://arxiv.org/abs/2609.36576) | Refining indirect prompt injection fragments | +| [LLM-Guided Prompt Evolution for Password Guessing](https://arxiv.org/abs/2604.12601) | Evolving prompts that maximize cracking rate | +| [Targeted Black-Box Attacks on Perceptual Hash Algorithms](https://arxiv.org/abs/2607.11472) | Evolutionary attack framework based on GigaEvo and OpenEvolve | -### **Research Grade** -Full reproducibility, extensive evaluation pipelines, and scientific rigor built-in. +**Building on OpenEvolve** -
+| Paper | Relationship | +|-------|--------------| +| [Hill Sampling for Test-Time Scaling](https://arxiv.org/abs/2609.25510) | Builds on OpenEvolve's problems and evaluators | +| [CodeEvolve: Runtime-Enriched Target Selection](https://arxiv.org/abs/2605.04677) | Extends OpenEvolve for Java and Salesforce Apex | +| [GraphIR](https://arxiv.org/abs/2608.01633) | Neural architecture search states integrated into OpenEvolve | -**OpenEvolve vs Manual Optimization:** +New evolutionary agents also routinely benchmark against OpenEvolve, including [HarnessTTT](https://arxiv.org/abs/2610.17370), [FrugalEvo](https://arxiv.org/abs/2610.03675), [SEDIMA](https://arxiv.org/abs/2610.02361), [EvoDuet](https://arxiv.org/abs/2609.40340), [LoongFlow](https://arxiv.org/abs/2512.24077) and [AscendOptimizer](https://arxiv.org/abs/2603.23566). -| Aspect | Manual Optimization | OpenEvolve | -|--------|-------------------|------------| -| **Time to Solution** | Days to weeks | Hours | -| **Exploration Breadth** | Limited by human creativity | Unlimited LLM creativity | -| **Reproducibility** | Hard to replicate | Fully deterministic | -| **Multi-objective** | Complex tradeoffs | Automatic Pareto optimization | -| **Scaling** | Doesn't scale | Parallel evolution across islands | +Using OpenEvolve in your work? Open a pull request to add it here, and see [Citation](#citation). -## Proven Achievements +## Results -
+| Domain | Achievement | Example | +|--------|-------------|---------| +| GPU optimization | Hardware-optimized kernel discovery, with 2-3x speedups on real hardware | [MLX Metal Kernels](examples/mlx_metal_kernel_opt/) | +| Mathematics | State-of-the-art circle packing (n=26) | [Circle Packing](examples/circle_packing/) | +| Algorithm design | Adaptive sorting algorithms | [Rust Adaptive Sort](examples/rust_adaptive_sort/) | +| Scientific computing | Automated filter design | [Signal Processing](examples/signal_processing/) | +| Multi-language | Python, Rust, R, Metal shaders | [All Examples](examples/) | -| **Domain** | **Achievement** | **Example** | -|---------------|-------------------|----------------| -| **GPU Optimization** | Hardware-optimized kernel discovery | [MLX Metal Kernels](examples/mlx_metal_kernel_opt/) | -| **Mathematical** | State-of-the-art circle packing (n=26) | [Circle Packing](examples/circle_packing/) | -| **Algorithm Design** | Adaptive sorting algorithms | [Rust Adaptive Sort](examples/rust_adaptive_sort/) | -| **Scientific Computing** | Automated filter design | [Signal Processing](examples/signal_processing/) | -| **Multi-Language** | Python, Rust, R, Metal shaders | [All Examples](examples/) | +| Generation 1 | Generation 190 | Generation 460 (final) | +|--------------|----------------|------------------------| +| ![Initial](examples/circle_packing/circle_packing_1.png) | ![Progress](examples/circle_packing/circle_packing_190.png) | ![Final](examples/circle_packing/circle_packing_460.png) | +| Random placement | Learning structure | State-of-the-art result | -
+**Compared with manual optimization:** -## 🚀 Quick Start +| Aspect | Manual Optimization | OpenEvolve | +|--------|---------------------|------------| +| Time to solution | Days to weeks | Hours | +| Exploration breadth | Limited by human creativity | Unlimited LLM creativity | +| Reproducibility | Hard to replicate | Fully deterministic | +| Multi-objective | Complex tradeoffs | Automatic Pareto optimization | +| Scaling | Doesn't scale | Parallel evolution across islands | + +## Quick Start -Get from zero to evolving code in **30 seconds**: +Requires Python 3.10+ and access to an LLM. ```bash -# Install OpenEvolve -pip install openevolve +git clone https://github.com/algorithmicsuperintelligence/openevolve.git +cd openevolve +pip install -e . # The example uses Google Gemini by default (free tier available) # Get your API key from: https://aistudio.google.com/apikey -export OPENAI_API_KEY="your-gemini-api-key" # Yes, use OPENAI_API_KEY env var +export OPENAI_API_KEY="your-gemini-api-key" # Yes, use OPENAI_API_KEY -# Run your first evolution! python openevolve-run.py examples/function_minimization/initial_program.py \ examples/function_minimization/evaluator.py \ --config examples/function_minimization/config.yaml \ --iterations 50 - -# Running several evolutions in parallel? Give each run a logger prefix so its -# log output is distinguishable (logs as "run1.openevolve.*") -python openevolve-run.py examples/function_minimization/initial_program.py \ - examples/function_minimization/evaluator.py \ - --logger-prefix run1 ``` -**Note:** The example config uses Gemini by default, but you can use any OpenAI-compatible provider by modifying the `config.yaml`. See the [configs](configs/) for full configuration options. - -### **Library Usage** +To use another provider, change `api_base` and the model in `config.yaml`. See [LLM Providers](docs/llm-providers.md). -OpenEvolve can be used as a library without any external files: +Or use it as a library (`pip install openevolve`): ```python -from openevolve import run_evolution, evolve_function +from openevolve import evolve_function -# Evolution with inline code (no files needed!) -result = run_evolution( - initial_program=''' - def fibonacci(n): - if n <= 1: return n - return fibonacci(n-1) + fibonacci(n-2) - ''', - evaluator=lambda path: {"score": benchmark_fib(path)}, - iterations=100 -) - -# Evolve Python functions directly def bubble_sort(arr): for i in range(len(arr)): - for j in range(len(arr)-1): - if arr[j] > arr[j+1]: - arr[j], arr[j+1] = arr[j+1], arr[j] + for j in range(len(arr) - 1): + if arr[j] > arr[j + 1]: + arr[j], arr[j + 1] = arr[j + 1], arr[j] return arr -if __name__ == '__main__': +if __name__ == "__main__": result = evolve_function( bubble_sort, - test_cases=[([3,1,2], [1,2,3]), ([5,2,8], [2,5,8])], - iterations=50 + test_cases=[([3, 1, 2], [1, 2, 3]), ([5, 2, 8], [2, 5, 8])], + iterations=50, ) - print(f"Evolved sorting algorithm: {result.best_code}") -``` - -> **Note:** On macOS and Windows, Python uses `spawn` for multiprocessing. You must wrap evolution calls in `if __name__ == '__main__':` to avoid subprocess bootstrap errors. -To experiment with island scheduling, pass an optional selector to `run_evolution`: - -```python -def select_island(context): - # Schedule the next iteration on the island with the fewest pending tasks. - return min(range(len(context.islands)), key=lambda i: context.pending_counts[i]) - -result = run_evolution( - initial_program="program.py", - evaluator="evaluator.py", - island_selector=select_island, -) -``` - -The selector receives a read-only `IslandSelectionContext` with the iteration number, -pending task counts, and each island's population size, best and average score, diversity, and -generation. It must return an island ID from `0` to `num_islands - 1`. It runs in the -main process before each iteration is submitted. Without a selector, OpenEvolve uses -its existing balanced island scheduling. - -Population management can also use decision hooks. For example, this policy lets a -candidate replace an island's MAP-Elites cell occupant when their scores tie: - -```python -from openevolve import run_evolution -from openevolve.population import PopulationStrategy - -strategy = PopulationStrategy( - replace_cell=lambda state, candidate, incumbent, island: ( - candidate.metrics["combined_score"] >= incumbent.metrics["combined_score"] - ), -) -result = run_evolution("program.py", "evaluator.py", population_strategy=strategy) -``` - -Other optional hooks are `admit(state, candidate, island) -> bool`, -`archive(state, candidate) -> ArchiveDecision`, -`evict(state, count, protected_ids) -> sequence of program IDs`, -`migration_due(state) -> bool`, and `migrate(state) -> sequence of MigrationMove`. -`state` is a detached `PopulationSnapshot` of programs, island memberships, cell owners, -the archive, population limits, and migration generations. Unspecified hooks retain -the existing rules. `ProgramDatabase` validates decisions and applies all changes; -the strategy never receives the mutable database. The initial seed cannot be rejected. - -**Prefer Docker?** See the [Installation & Setup](#installation--setup) section for Docker options. - -## See It In Action - -
-Circle Packing: From Random to State-of-the-Art - -**Watch OpenEvolve discover optimal circle packing in real-time:** - -| Generation 1 | Generation 190 | Generation 460 (Final) | -|--------------|----------------|----------------------| -| ![Initial](examples/circle_packing/circle_packing_1.png) | ![Progress](examples/circle_packing/circle_packing_190.png) | ![Final](examples/circle_packing/circle_packing_460.png) | -| Random placement | Learning structure | **State-of-the-art result** | - -**Result**: Matches published benchmarks for n=26 circle packing problem. - -
- -
-GPU Kernel Evolution - -**Before (Baseline)**: -```metal -// Standard attention implementation -kernel void attention_baseline(/* ... */) { - // Generic matrix multiplication - float sum = 0.0; - for (int i = 0; i < seq_len; i++) { - sum += query[tid] * key[i]; - } -} + print(result.best_code) ``` -**After Evolution (2.8x faster)**: -```metal -// OpenEvolve discovered optimization -kernel void attention_evolved(/* ... */) { - // Hardware-aware tiling + unified memory optimization - threadgroup float shared_mem[256]; - // ... evolved algorithm exploiting Apple Silicon architecture -} -``` - -**Performance Impact**: 2.8x speedup on Apple M1 Pro, maintaining numerical accuracy. - -
- -## How OpenEvolve Works - -OpenEvolve implements a sophisticated **evolutionary coding pipeline** that goes far beyond simple optimization: - -![OpenEvolve Architecture](openevolve-architecture.png) - -### **Core Innovation**: MAP-Elites + LLMs - -- **Quality-Diversity Evolution**: Maintains diverse populations across feature dimensions -- **Island-Based Architecture**: Multiple populations prevent premature convergence -- **LLM Ensemble**: Multiple models with intelligent fallback strategies -- **Artifact Side-Channel**: Error feedback improves subsequent generations - -### **Advanced Features** - -
-Scientific Reproducibility - -- **Comprehensive Seeding**: Every component (LLM, database, evaluation) is seeded -- **Default Seed=42**: Immediate reproducible results out of the box -- **Deterministic Evolution**: Exact reproduction of runs across machines -- **Component Isolation**: Hash-based isolation prevents cross-contamination - -
- -
-Advanced LLM Integration - -- **Universal API**: Works with OpenAI, Google, Claude Code CLI, local models, and proxies -- **Intelligent Ensembles**: Weighted combinations with sophisticated fallback -- **Test-Time Compute**: Enhanced reasoning through proxy systems (see [OptiLLM setup](#llm-provider-setup)) -- **Plugin Ecosystem**: Support for advanced reasoning plugins - -
- -
-Evolution Algorithm Innovations - -- **Double Selection**: Different programs for performance vs inspiration -- **Adaptive Feature Dimensions**: Custom quality-diversity metrics -- **Migration Patterns**: Ring topology with controlled gene flow -- **Multi-Strategy Sampling**: Elite, diverse, and exploratory selection +`run_evolution` also takes inline code and an evaluator function. The `__main__` guard is required on macOS and Windows, where multiprocessing uses `spawn`. -
- -## Perfect For - -| **Use Case** | **Why OpenEvolve Excels** | -|--------------|---------------------------| -| **Performance Optimization** | Discovers hardware-specific optimizations humans miss | -| **Algorithm Discovery** | Finds novel approaches to classic problems | -| **Scientific Computing** | Automates tedious manual tuning processes | -| **Competitive Programming** | Generates multiple solution strategies | -| **Multi-Objective Problems** | Pareto-optimal solutions across dimensions | - -## 🛠 Installation & Setup - -### Requirements -- **Python**: 3.10+ -- **LLM Access**: Any OpenAI-compatible API, or [Claude Code CLI](https://docs.anthropic.com/en/docs/claude-code) -- **Optional**: Docker for containerized runs - -### Installation Options - -
-📦 PyPI (Recommended) +A Docker image is available too: ```bash -pip install openevolve -``` - -
- -
-🔧 Development Install - -```bash -git clone https://github.com/algorithmicsuperintelligence/openevolve.git -cd openevolve -pip install -e ".[dev]" -``` - -
- -
-🐳 Docker - -```bash -# Pull the image docker pull ghcr.io/algorithmicsuperintelligence/openevolve:latest -# Run an example docker run --rm -v $(pwd):/app ghcr.io/algorithmicsuperintelligence/openevolve:latest \ examples/function_minimization/initial_program.py \ examples/function_minimization/evaluator.py --iterations 100 ``` -
- -### Cost Estimation - -**Cost depends on your LLM provider and iterations:** - -- **o3**: ~$0.15-0.60 per iteration (depending on code size) -- **o3-mini**: ~$0.03-0.12 per iteration (more cost-effective) -- **Gemini-2.5-Pro**: ~$0.08-0.30 per iteration -- **Gemini-2.5-Flash**: ~$0.01-0.05 per iteration (fastest and cheapest) -- **Local models**: Nearly free after setup -- **OptiLLM**: Use cheaper models with test-time compute for better results - -**Cost-saving tips:** -- Start with fewer iterations (100-200) -- Use o3-mini, Gemini-2.5-Flash or local models for exploration -- Use cascade evaluation to filter bad programs early -- Configure smaller population sizes initially - -### LLM Provider Setup - -OpenEvolve works with **any OpenAI-compatible API**: - -
-🔥 OpenAI (Direct) - -```bash -export OPENAI_API_KEY="sk-..." -# Uses OpenAI endpoints by default -``` - -
- -
-🤖 Google Gemini - -```yaml -# config.yaml -llm: - api_base: "https://generativelanguage.googleapis.com/v1beta/openai/" - primary_model: "gemini-2.5-pro" -``` - -```bash -export OPENAI_API_KEY="your-gemini-api-key" -``` - -
- -
-🏠 Local Models (Ollama/vLLM) - -```yaml -# config.yaml -llm: - api_base: "http://localhost:11434/v1" # Ollama - primary_model: "codellama:7b" -``` - -
- -
-⚡ OptiLLM (Advanced) - -For maximum flexibility with rate limiting, model routing, and test-time compute: - -```bash -# Install OptiLLM -pip install optillm - -# Start OptiLLM proxy -optillm --port 8000 - -# Point OpenEvolve to OptiLLM -export OPENAI_API_KEY="your-actual-key" -``` - -```yaml -llm: - api_base: "http://localhost:8000/v1" - primary_model: "moa&readurls-o3" # Test-time compute + web access -``` - -
- -
-🔮 Claude Code CLI (No API Key) - -Use the [Claude Code CLI](https://docs.anthropic.com/en/docs/claude-code) as the LLM backend — no API keys needed, authentication uses the CLI's OAuth session. - -```bash -# Install and authenticate -npm install -g @anthropic-ai/claude-code -claude login -``` - -```yaml -# config.yaml -llm: - provider: "claude_code" - models: - - name: "sonnet" - weight: 0.8 - max_tokens: 16000 - max_budget_usd: 1.0 - - name: "haiku" - weight: 0.2 - max_tokens: 8000 -``` - -See the [Claude Code quickstart example](examples/claude_code_quickstart/) for a complete walkthrough. - -
- -
-🤖 GitHub Copilot CLI (No API Key) - -Use the [GitHub Copilot CLI](https://docs.github.com/copilot/how-tos/copilot-cli) as the LLM backend — no API keys needed, authentication uses your GitHub Copilot subscription. - -```bash -# Install and authenticate -npm install -g @github/copilot -copilot login -``` - -```yaml -# config.yaml -llm: - provider: "copilot_cli" - models: - - name: "claude-sonnet-4.6" - weight: 0.8 - reasoning_effort: "medium" - - name: "gpt-5.4" - weight: 0.2 -``` - -See the [Copilot CLI quickstart example](examples/copilot_cli_quickstart/) for a complete walkthrough. - -
- -
-🧩 Codex App Server (No API Key) - -Use the [Codex CLI's app-server](https://developers.openai.com/codex/app-server/) as the LLM backend — no API keys needed. Authentication uses your ChatGPT (Codex) subscription: the app-server owns the managed browser/device-code login and refreshes the tokens itself. OpenEvolve never reads the CLI's token files, and before each generation it verifies with the app-server that a ChatGPT-plan login is active (it fails with a clear message pointing at `codex login` otherwise). To bill an API key instead, keep the default OpenAI provider — both paths coexist. - -```bash -# Install and authenticate -npm install -g @openai/codex -codex login # browser flow, or: codex login --device-code -``` - -```yaml -# config.yaml -llm: - provider: "codex_app_server" - models: - - name: "gpt-5.3-codex" - weight: 0.8 - reasoning_effort: "medium" - - name: "gpt-5.3-codex-mini" - weight: 0.2 -``` - -Each call speaks newline-delimited JSON-RPC to a short-lived `codex app-server` process over stdio (unsupported on Windows). Threads run non-interactive: tool approvals are `never` and the sandbox is read-only, so responses are generated without mutating your workspace. - -
- -## Examples Gallery - -To use unchanged local evaluators from Google's AlphaEvolve examples, see the -[AlphaEvolve evaluator interoperability walkthrough](examples/alphaevolve_interop/). - -
- -### **Showcase Projects** - -| Project | Domain | Achievement | Demo | -|---------|--------|-------------|------| -| [**Function Minimization**](examples/function_minimization/) | Optimization | Random → Simulated Annealing | [View Results](examples/function_minimization/openevolve_output/) | -| [**MLX GPU Kernels**](examples/mlx_metal_kernel_opt/) | Hardware | Apple Silicon optimization | [Benchmarks](examples/mlx_metal_kernel_opt/README.md) | -| [**Rust Adaptive Sort**](examples/rust_adaptive_sort/) | Algorithms | Data-aware sorting | [Code Evolution](examples/rust_adaptive_sort/) | -| [**Symbolic Regression**](examples/symbolic_regression/) | Science | Automated equation discovery | [LLM-SRBench](examples/symbolic_regression/) | -| [**Web Scraper + OptiLLM**](examples/web_scraper_optillm/) | AI Integration | Test-time compute optimization | [Smart Scraping](examples/web_scraper_optillm/) | - -
- -### **Quick Example**: Function Minimization - -**Watch OpenEvolve evolve from random search to sophisticated optimization:** - -```python -# Initial Program (Random Search) -def minimize_function(func, bounds, max_evals=1000): - best_x, best_val = None, float('inf') - for _ in range(max_evals): - x = random_point_in_bounds(bounds) - val = func(x) - if val < best_val: - best_x, best_val = x, val - return best_x, best_val -``` - -**Evolution Process** - -```python -# Evolved Program (Simulated Annealing + Adaptive Cooling) -def minimize_function(func, bounds, max_evals=1000): - x = random_point_in_bounds(bounds) - temp = adaptive_initial_temperature(func, bounds) - - for i in range(max_evals): - neighbor = generate_neighbor(x, temp, bounds) - delta = func(neighbor) - func(x) - - if delta < 0 or random.random() < exp(-delta/temp): - x = neighbor - - temp *= adaptive_cooling_rate(i, max_evals) # Dynamic cooling - - return x, func(x) -``` - -**Performance**: 100x improvement in convergence speed! +## How It Works -### **Advanced Examples** - -
-Prompt Evolution - -**Evolve prompts instead of code** for better LLM performance. See the [LLM Prompt Optimization example](examples/llm_prompt_optimization/) for a complete case study with HotpotQA achieving +23% accuracy improvement. - -[Full Example](examples/llm_prompt_optimization/) - -
- -
-🏁 Competitive Programming - -**Automatic solution generation** for programming contests: - -```python -# Problem: Find maximum subarray sum -# OpenEvolve discovers multiple approaches: - -# Evolution Path 1: Brute Force → Kadane's Algorithm -# Evolution Path 2: Divide & Conquer → Optimized Kadane's -# Evolution Path 3: Dynamic Programming → Space-Optimized DP -``` - -[Online Judge Integration](examples/online_judge_programming/) - -
- -## Configuration - -OpenEvolve offers extensive configuration for advanced users: - -```yaml -# Advanced Configuration Example -max_iterations: 1000 -random_seed: 42 # Full reproducibility - -# Optional prefix for all logger names, e.g. "run1" logs as "run1.openevolve.*"; -# useful to tell apart log output from multiple concurrent runs -logger_prefix: "run1" - -llm: - # Ensemble configuration - models: - - name: "gemini-2.5-pro" - weight: 0.6 - - name: "gemini-2.5-flash" - weight: 0.4 - temperature: 0.7 - -database: - # MAP-Elites quality-diversity - population_size: 500 - num_islands: 5 # Parallel evolution - # island_initial_programs: [null, "seeds/alt.py"] # Optional per-island starting programs - migration_interval: 20 - feature_dimensions: ["complexity", "diversity", "performance"] - - # Optional novelty filtering with Gemini embeddings - embedding_model: "gemini-embedding-001" - similarity_threshold: 0.99 - # Or any OpenAI-compatible embedding endpoint (OpenRouter, local servers): - # embedding_api_base: "https://openrouter.ai/api/v1" # or OPENAI_EMBEDDING_BASE_URL - -evaluator: - enable_artifacts: true # Error feedback to LLM - cascade_evaluation: true # Multi-stage testing - use_llm_feedback: true # AI code quality assessment - evaluator_args: # Keyword arguments forwarded to evaluate() - weight: 2.0 - -prompt: - # Sophisticated inspiration system - num_top_programs: 3 # Best performers - num_diverse_programs: 2 # Creative exploration - include_artifacts: true # Execution feedback - - # Custom templates - template_dir: "custom_prompts/" - use_template_stochasticity: true # Randomized prompts -``` - -For Gemini embeddings, set `GEMINI_API_KEY`. `GOOGLE_API_KEY` is also supported -as a fallback. OpenEvolve uses Google's OpenAI-compatible endpoint automatically. - -
-🎯 Feature Engineering - -**Control how programs are organized in the quality-diversity grid:** - -```yaml -database: - feature_dimensions: - - "complexity" # Built-in: code length - - "diversity" # Built-in: structural diversity - - "performance" # Custom: from your evaluator - - "memory_usage" # Custom: from your evaluator - - feature_bins: - complexity: 10 # 10 complexity levels - performance: 20 # 20 performance buckets - memory_usage: 15 # 15 memory usage categories -``` - -**Important**: Return raw values from evaluator, OpenEvolve handles binning automatically. - -
- -
-📞 Passing Arguments to Your Evaluator - -If your `evaluate()` function takes extra keyword arguments beyond the program -path, pass them with the `--evaluator-args` CLI flag (a JSON object): - -```bash -openevolve-run initial_program.py evaluator.py \ - --evaluator-args '{"weight": 2.0, "label": "experiment-1"}' -``` - -or configure them in YAML: - -```yaml -evaluator: - evaluator_args: - weight: 2.0 - label: "experiment-1" -``` - -OpenEvolve inspects the signature of your `evaluate()` function: arguments are -forwarded only when the function accepts them (named parameters or `**kwargs`), -and a startup error lists the offending keys and the actual signature if it -does not — so typos fail fast instead of silently doing nothing. Evaluators -with a single `program_path` argument are unaffected. - -This replaces the earlier workaround of smuggling per-run settings to the -evaluator through environment variables. (Note: custom top-level keys in -`config.yaml` are dropped silently by the config parser — use -`evaluator.evaluator_args` or the CLI flag instead.) - -
- -
-🎨 Custom Prompt Templates - -**Advanced prompt engineering** with custom templates: - -```yaml -prompt: - template_dir: "custom_templates/" - use_template_stochasticity: true - template_variations: - greeting: - - "Let's enhance this code:" - - "Time to optimize:" - - "Improving the algorithm:" - improvement_suggestion: - - "Here's how we could improve this code:" - - "I suggest the following improvements:" - - "We can enhance this code by:" -``` - -**How it works:** Place `{greeting}` or `{improvement_suggestion}` placeholders in your templates, and OpenEvolve will randomly choose from the variations for each generation, adding diversity to prompts. - -See [prompt examples](examples/llm_prompt_optimization/templates/) for complete template customization. - -
- -## Crafting Effective System Messages - -**System messages are the secret to successful evolution.** They guide the LLM's understanding of your domain, constraints, and optimization goals. A well-crafted system message can be the difference between random mutations and targeted improvements. - -### Why System Messages Matter - -The system message in your config.yaml is arguably the most important component for evolution success: - -- **Domain Expertise**: Provides LLM with specific knowledge about your problem space -- **Constraint Awareness**: Defines what can and cannot be changed during evolution -- **Optimization Focus**: Guides the LLM toward meaningful improvements -- **Error Prevention**: Helps avoid common pitfalls and compilation errors - -### The Iterative Creation Process - -Based on successful OpenEvolve implementations, system messages are best created through iteration: - -
-🔄 Step-by-Step Process - -**Phase 1: Initial Draft** - -1. Start with a basic system message describing your goal -2. Run 20-50 evolution iterations to observe behavior -3. Note where the system gets "stuck" or makes poor choices - -**Phase 2: Refinement** - -4. Add specific guidance based on observed issues -5. Include domain-specific terminology and concepts -6. Define clear constraints and optimization targets -7. Run another batch of iterations - -**Phase 3: Specialization** - -8. Add detailed examples of good vs bad approaches -9. Include specific library/framework guidance -10. Add error avoidance patterns you've observed -11. Fine-tune based on artifact feedback - -**Phase 4: Optimization** - -12. Consider using OpenEvolve itself to optimize your prompt -13. Measure improvements using combined score metrics - -
- -### Examples by Complexity - -#### **Simple: General Optimization** -```yaml -prompt: - system_message: | - You are an expert programmer specializing in optimization algorithms. - Your task is to improve a function minimization algorithm to find the - global minimum reliably, escaping local minima that might trap simple algorithms. -``` - -#### **Intermediate: Domain-Specific Guidance** -```yaml -prompt: - system_message: | - You are an expert prompt engineer. Your task is to revise prompts for LLMs. - - Your improvements should: - * Clarify vague instructions and eliminate ambiguity - * Strengthen alignment between prompt and desired task outcome - * Improve robustness against edge cases - * Include formatting instructions and examples where helpful - * Avoid unnecessary verbosity - - Return only the improved prompt text without explanations. -``` - -#### ⚡ **Advanced: Hardware-Specific Optimization** -```yaml -prompt: - system_message: | - You are an expert Metal GPU programmer specializing in custom attention - kernels for Apple Silicon. - - # TARGET: Optimize Metal Kernel for Grouped Query Attention (GQA) - # HARDWARE: Apple M-series GPUs with unified memory architecture - # GOAL: 5-15% performance improvement - - # OPTIMIZATION OPPORTUNITIES: - **1. Memory Access Pattern Optimization:** - - Coalesced access patterns for Apple Silicon - - Vectorized loading using SIMD - - Pre-compute frequently used indices - - **2. Algorithm Fusion:** - - Combine max finding with score computation - - Reduce number of passes through data - - # CONSTRAINTS - CRITICAL SAFETY RULES: - **MUST NOT CHANGE:** - ❌ Kernel function signature - ❌ Template parameter names or types - ❌ Overall algorithm correctness - - **ALLOWED TO OPTIMIZE:** - ✅ Memory access patterns and indexing - ✅ Computation order and efficiency - ✅ Vectorization and SIMD utilization - ✅ Apple Silicon specific optimizations -``` - -### Best Practices - -
-🎨 Prompt Engineering Patterns - -**Structure Your Message:** Start with role definition → Define task/context → List optimization opportunities → Set constraints → Success criteria - -**Use Specific Examples:** -```yaml -# Good: "Focus on reducing memory allocations. Example: Replace `new Vector()` with pre-allocated arrays." -# Avoid: "Make the code faster" -``` - -**Include Domain Knowledge:** -```yaml -# Good: "For GPU kernels: 1) Memory coalescing 2) Occupancy 3) Shared memory usage" -# Avoid: "Optimize the algorithm" -``` - -**Set Clear Boundaries:** -```yaml -system_message: | - MUST NOT CHANGE: ❌ Function signatures ❌ Algorithm correctness ❌ External API - ALLOWED: ✅ Internal implementation ✅ Data structures ✅ Performance optimizations -``` - -
- -
-🔬 Advanced Techniques - -**Artifact-Driven Iteration:** Enable artifacts in config → Include common error patterns in system message → Add guidance based on stderr/warning patterns - -**Multi-Phase Evolution:** Start broad ("Explore different algorithmic approaches"), then focus ("Given successful simulated annealing, focus on parameter tuning") - -**Template Stochasticity:** See the [Configuration section](#configuration) for complete template variation examples. - -
- -### Meta-Evolution: Using OpenEvolve to Optimize Prompts - -**You can use OpenEvolve to evolve your system messages themselves!** This powerful technique lets you optimize prompts for better LLM performance automatically. - -See the [LLM Prompt Optimization example](examples/llm_prompt_optimization/) for a complete implementation, including the HotpotQA case study with +23% accuracy improvement. - -### Common Pitfalls to Avoid - -- **Too Vague**: "Make the code better" → Specify exactly what "better" means -- **Too Restrictive**: Over-constraining can prevent useful optimizations -- **Missing Context**: Include relevant domain knowledge and terminology -- **No Examples**: Concrete examples guide LLM better than abstract descriptions -- **Ignoring Artifacts**: Don't refine prompts based on error feedback - -## Artifacts & Debugging - -**Artifacts side-channel** provides rich feedback to accelerate evolution: - -```python -# Evaluator can return execution context -from openevolve.evaluation_result import EvaluationResult - -return EvaluationResult( - metrics={"performance": 0.85, "correctness": 1.0}, - artifacts={ - "stderr": "Warning: suboptimal memory access pattern", - "profiling_data": {...}, - "llm_feedback": "Code is correct but could use better variable names", - "build_warnings": ["unused variable x"] - } -) -``` - -**Next generation prompt automatically includes:** - -```markdown -## Previous Execution Feedback -⚠️ Warning: suboptimal memory access pattern -💡 LLM Feedback: Code is correct but could use better variable names -🔧 Build Warnings: unused variable x -``` - -This creates a **feedback loop** where each generation learns from previous mistakes! - -## Visualization - -**Real-time evolution tracking** with interactive web interface: - -```bash -# Install visualization dependencies -pip install -r scripts/requirements.txt - -# Launch interactive visualizer -python scripts/visualizer.py - -# Or visualize specific checkpoint -python scripts/visualizer.py --path examples/function_minimization/openevolve_output/checkpoints/checkpoint_100/ -``` - -**Features:** - -- 🌳 **Evolution tree** with parent-child relationships -- 📈 **Performance tracking** across generations -- 🔍 **Code diff viewer** showing mutations -- 📊 **MAP-Elites grid** visualization -- 🎯 **Multi-metric analysis** with custom dimensions - -![OpenEvolve Visualizer](openevolve-visualizer.png) - -## Roadmap - -### **🔥 Upcoming Features** - -- [ ] **Multi-Modal Evolution**: Images, audio, and text simultaneously -- [ ] **Federated Learning**: Distributed evolution across multiple machines -- [ ] **AutoML Integration**: Hyperparameter and architecture evolution -- [ ] **Benchmark Suite**: Standardized evaluation across domains - -### **🌟 Research Directions** - -- [ ] **Self-Modifying Prompts**: Evolution modifies its own prompting strategy -- [ ] **Cross-Language Evolution**: Python → Rust → C++ optimization chains -- [ ] **Neurosymbolic Reasoning**: Combine neural and symbolic approaches -- [ ] **Human-AI Collaboration**: Interactive evolution with human feedback - -Want to contribute? Check out our [roadmap discussions](https://github.com/algorithmicsuperintelligence/openevolve/discussions/categories/roadmap)! - -## FAQ - -
-💰 How much does it cost to run? - -See the [Cost Estimation](#cost-estimation) section in Installation & Setup for detailed pricing information and cost-saving tips. - -
- -
-🆚 How does this compare to manual optimization? - -| Aspect | Manual | OpenEvolve | -|--------|--------|------------| -| **Initial Learning** | Weeks to understand domain | Minutes to start | -| **Solution Quality** | Depends on expertise | Consistently explores novel approaches | -| **Time Investment** | Days-weeks per optimization | Hours for complete evolution | -| **Reproducibility** | Hard to replicate exact process | Perfect reproduction with seeds | -| **Scaling** | Doesn't scale beyond human capacity | Parallel evolution across islands | - -**OpenEvolve shines** when you need to explore large solution spaces or optimize for multiple objectives simultaneously. - -
- -
-🔧 Can I use my own LLM? - -**Yes!** OpenEvolve supports any OpenAI-compatible API: - -- **Commercial**: OpenAI, Google, Cohere -- **Local**: Ollama, vLLM, LM Studio, text-generation-webui -- **Advanced**: OptiLLM for routing and test-time compute - -Just set the `api_base` in your config to point to your endpoint. - -
+![OpenEvolve Architecture](openevolve-architecture.png) -
-🚨 What if evolution gets stuck? +- **Quality-diversity evolution:** MAP-Elites keeps diverse programs across feature dimensions. +- **Island populations:** separate populations with periodic migration prevent premature convergence. +- **LLM ensembles:** weighted combinations of models, from any OpenAI-compatible API or the Claude Code, GitHub Copilot and Codex CLIs. +- **Artifact feedback:** errors and evaluator output feed the next generation's prompt. +- **Reproducibility:** every component is seeded. -**Built-in mechanisms prevent stagnation:** +Mark the code to evolve with `EVOLVE-BLOCK-START` and `EVOLVE-BLOCK-END` comments. More in [How It Works](docs/how-it-works.md). -- **Island migration**: Fresh genes from other populations -- **Temperature control**: Exploration vs exploitation balance -- **Diversity maintenance**: MAP-Elites prevents convergence -- **Artifact feedback**: Error messages guide improvements -- **Template stochasticity**: Randomized prompts break patterns +Good fits are performance optimization, algorithm discovery, scientific computing, competitive programming and multi-objective problems. -**Manual interventions:** -- Increase `num_diverse_programs` for more exploration -- Add custom feature dimensions to diversify search -- Use template variations to randomize prompts -- Adjust migration intervals for more cross-pollination +## Examples -
+| Project | Domain | What it evolves | +|---------|--------|-----------------| +| [Function Minimization](examples/function_minimization/) | Optimization | Random search to simulated annealing | +| [Circle Packing](examples/circle_packing/) | Mathematics | The n=26 AlphaEvolve task | +| [AlphaEvolve Math Problems](examples/alphaevolve_math_problems/) | Mathematics | Problems from AlphaEvolve's appendices | +| [MLX GPU Kernels](examples/mlx_metal_kernel_opt/) | Hardware | Apple Silicon Metal kernels | +| [Rust Adaptive Sort](examples/rust_adaptive_sort/) | Algorithms | Data-aware sorting | +| [Symbolic Regression](examples/symbolic_regression/) | Science | Equation discovery on LLM-SRBench | +| [Prompt Optimization](examples/llm_prompt_optimization/) | LLMs | Prompts instead of code | +| [Online Judge](examples/online_judge_programming/) | Programming | Competitive programming solutions | +| [Web Scraper + OptiLLM](examples/web_scraper_optillm/) | AI integration | Test-time compute | -
-📈 How do I measure success? +Running Google's AlphaEvolve examples? See the [interoperability walkthrough](examples/alphaevolve_interop/). -**Multiple success metrics:** +## Documentation -1. **Primary Metric**: Your evaluator's `combined_score` or metric average -2. **Convergence**: Best score improvement over time -3. **Diversity**: MAP-Elites grid coverage -4. **Efficiency**: Iterations to reach target performance -5. **Robustness**: Performance across different test cases +- [How It Works](docs/how-it-works.md): the evolution pipeline and its features +- [LLM Providers](docs/llm-providers.md): provider setup and cost +- [Configuration](docs/configuration.md): options, evaluator arguments, prompt templates, island and population strategies, artifacts, and the visualizer +- [System Messages](docs/system-messages.md): the guidance that matters most for good results +- [Showcase](docs/showcase.md): worked before-and-after examples +- [FAQ](docs/faq.md) and [Roadmap](docs/roadmap.md) -**Use the visualizer** to track all metrics in real-time and identify when evolution has converged. +Articles: -
+- [Towards Open Evolutionary Agents](https://huggingface.co/blog/driaforall/towards-open-evolutionary-agents) +- [OpenEvolve: GPU Kernel Discovery](https://huggingface.co/blog/codelion/openevolve-gpu-kernel-discovery) +- [OpenEvolve: Evolutionary Coding with LLMs](https://huggingface.co/blog/codelion/openevolve) -### **Contributors** +## Contributing -Thanks to all our amazing contributors who make OpenEvolve possible! +Contributions are welcome. See the [Contributing Guide](CONTRIBUTING.md) and [`good first issue`](https://github.com/algorithmicsuperintelligence/openevolve/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) labels. -### **Contributing** - -We welcome contributions! Here's how to get started: - -1. 🍴 **Fork** the repository -2. 🌿 **Create** your feature branch: `git checkout -b feat-amazing-feature` -3. ✨ **Add** your changes and tests -4. ✅ **Test** everything: `python -m unittest discover tests` -5. 📝 **Commit** with a clear message -6. 🚀 **Push** and create a Pull Request - -**New to open source?** Check out our [Contributing Guide](CONTRIBUTING.md) and look for [`good-first-issue`](https://github.com/algorithmicsuperintelligence/openevolve/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) labels! - -### **Academic & Research** - -**Articles & Blog Posts About OpenEvolve**: -- [Towards Open Evolutionary Agents](https://huggingface.co/blog/driaforall/towards-open-evolutionary-agents) - Evolution of coding agents and the open-source movement -- [OpenEvolve: GPU Kernel Discovery](https://huggingface.co/blog/codelion/openevolve-gpu-kernel-discovery) - Automated discovery of optimized GPU kernels -- [OpenEvolve: Evolutionary Coding with LLMs](https://huggingface.co/blog/codelion/openevolve) - Introduction to evolutionary algorithm discovery using large language models - ## Citation If you use OpenEvolve in your research, please cite: @@ -1061,14 +199,3 @@ If you use OpenEvolve in your research, please cite: url = {https://github.com/algorithmicsuperintelligence/openevolve} } ``` ---- - -
- -### **🚀 Ready to evolve your code?** - -**Maintained by the OpenEvolve community** - -*If OpenEvolve helps you discover breakthrough algorithms, please consider starring this repository.* - -
diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000000..3845e56a46 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,257 @@ +# Configuration + +Every option, with its default, is in [`configs/default_config.yaml`](../configs/default_config.yaml). This page covers the ones people most often need. + +```yaml +# Advanced Configuration Example +max_iterations: 1000 +random_seed: 42 # Full reproducibility + +# Optional prefix for all logger names, e.g. "run1" logs as "run1.openevolve.*"; +# useful to tell apart log output from multiple concurrent runs +logger_prefix: "run1" + +llm: + # Ensemble configuration + models: + - name: "gemini-2.5-pro" + weight: 0.6 + - name: "gemini-2.5-flash" + weight: 0.4 + temperature: 0.7 + +database: + # MAP-Elites quality-diversity + population_size: 500 + num_islands: 5 # Parallel evolution + # island_initial_programs: [null, "seeds/alt.py"] # Optional per-island starting programs + migration_interval: 20 + feature_dimensions: ["complexity", "diversity", "performance"] + + # Optional novelty filtering with Gemini embeddings + embedding_model: "gemini-embedding-001" + similarity_threshold: 0.99 + # Or any OpenAI-compatible embedding endpoint (OpenRouter, local servers): + # embedding_api_base: "https://openrouter.ai/api/v1" # or OPENAI_EMBEDDING_BASE_URL + +evaluator: + enable_artifacts: true # Error feedback to LLM + cascade_evaluation: true # Multi-stage testing + use_llm_feedback: true # AI code quality assessment + evaluator_args: # Keyword arguments forwarded to evaluate() + weight: 2.0 + +prompt: + # Sophisticated inspiration system + num_top_programs: 3 # Best performers + num_diverse_programs: 2 # Creative exploration + include_artifacts: true # Execution feedback + + # Custom templates + template_dir: "custom_prompts/" + use_template_stochasticity: true # Randomized prompts +``` + +For Gemini embeddings, set `GEMINI_API_KEY`. `GOOGLE_API_KEY` is also supported +as a fallback. OpenEvolve uses Google's OpenAI-compatible endpoint automatically. + +## Command-Line Options + +```bash +openevolve-run initial_program.py evaluator.py \ + --config config.yaml \ + --iterations 100 \ + --target-score 0.95 \ + --checkpoint path/to/checkpoint_directory \ + --logger-prefix run1 +``` + +`--target-score` stops the run once `combined_score` reaches the value. `--checkpoint` resumes from a saved run. `--logger-prefix` keeps log output distinguishable across parallel runs (it logs as `run1.openevolve.*`). + +## Evolution Modes + +```yaml +diff_based_evolution: true # SEARCH/REPLACE diffs; false asks for a full rewrite every iteration +full_rewrite_probability: 0.2 # With diffs on, chance an iteration asks for a full rewrite instead +enforce_evolve_blocks: true # Revert edits outside the EVOLVE-BLOCK-START/END markers + +evolution_trace: + enabled: true + include_failures: true # Also log failed iterations to _failures.jsonl +``` + +## Feature Engineering + +**Control how programs are organized in the quality-diversity grid:** + +```yaml +database: + feature_dimensions: + - "complexity" # Built-in: code length + - "diversity" # Built-in: structural diversity + - "performance" # Custom: from your evaluator + - "memory_usage" # Custom: from your evaluator + + feature_bins: + complexity: 10 # 10 complexity levels + performance: 20 # 20 performance buckets + memory_usage: 15 # 15 memory usage categories +``` + +**Important**: Return raw values from evaluator, OpenEvolve handles binning automatically. + +## Passing Arguments to Your Evaluator + +If your `evaluate()` function takes extra keyword arguments beyond the program +path, pass them with the `--evaluator-args` CLI flag (a JSON object): + +```bash +openevolve-run initial_program.py evaluator.py \ + --evaluator-args '{"weight": 2.0, "label": "experiment-1"}' +``` + +or configure them in YAML: + +```yaml +evaluator: + evaluator_args: + weight: 2.0 + label: "experiment-1" +``` + +OpenEvolve inspects the signature of your `evaluate()` function: arguments are +forwarded only when the function accepts them (named parameters or `**kwargs`), +and a startup error lists the offending keys and the actual signature if it +does not — so typos fail fast instead of silently doing nothing. Evaluators +with a single `program_path` argument are unaffected. + +This replaces the earlier workaround of smuggling per-run settings to the +evaluator through environment variables. (Note: custom top-level keys in +`config.yaml` are dropped silently by the config parser — use +`evaluator.evaluator_args` or the CLI flag instead.) + +## Custom Prompt Templates + +**Advanced prompt engineering** with custom templates: + +```yaml +prompt: + template_dir: "custom_templates/" + use_template_stochasticity: true + template_variations: + greeting: + - "Let's enhance this code:" + - "Time to optimize:" + - "Improving the algorithm:" + improvement_suggestion: + - "Here's how we could improve this code:" + - "I suggest the following improvements:" + - "We can enhance this code by:" +``` + +**How it works:** Place `{greeting}` or `{improvement_suggestion}` placeholders in your templates, and OpenEvolve will randomly choose from the variations for each generation, adding diversity to prompts. + +See [prompt examples](../examples/llm_prompt_optimization/templates/) for complete template customization. + +## Island Scheduling + +Pass an optional selector to `run_evolution`: + +```python +def select_island(context): + # Schedule the next iteration on the island with the fewest pending tasks. + return min(range(len(context.islands)), key=lambda i: context.pending_counts[i]) + +result = run_evolution( + initial_program="program.py", + evaluator="evaluator.py", + island_selector=select_island, +) +``` + +The selector receives a read-only `IslandSelectionContext` with the iteration number, +pending task counts, and each island's population size, best and average score, diversity, and +generation. It must return an island ID from `0` to `num_islands - 1`. It runs in the +main process before each iteration is submitted. Without a selector, OpenEvolve uses +its existing balanced island scheduling. + +## Population Strategies + +Population management can use decision hooks. For example, this policy lets a +candidate replace an island's MAP-Elites cell occupant when their scores tie: + +```python +from openevolve import run_evolution +from openevolve.population import PopulationStrategy + +strategy = PopulationStrategy( + replace_cell=lambda state, candidate, incumbent, island: ( + candidate.metrics["combined_score"] >= incumbent.metrics["combined_score"] + ), +) +result = run_evolution("program.py", "evaluator.py", population_strategy=strategy) +``` + +Other optional hooks are `admit(state, candidate, island) -> bool`, +`archive(state, candidate) -> ArchiveDecision`, +`evict(state, count, protected_ids) -> sequence of program IDs`, +`migration_due(state) -> bool`, and `migrate(state) -> sequence of MigrationMove`. +`state` is a detached `PopulationSnapshot` of programs, island memberships, cell owners, +the archive, population limits, and migration generations. Unspecified hooks retain +the existing rules. `ProgramDatabase` validates decisions and applies all changes; +the strategy never receives the mutable database. The initial seed cannot be rejected. + +## Artifacts + +**Artifacts side-channel** provides rich feedback to accelerate evolution: + +```python +# Evaluator can return execution context +from openevolve.evaluation_result import EvaluationResult + +return EvaluationResult( + metrics={"performance": 0.85, "correctness": 1.0}, + artifacts={ + "stderr": "Warning: suboptimal memory access pattern", + "profiling_data": {...}, + "llm_feedback": "Code is correct but could use better variable names", + "build_warnings": ["unused variable x"] + } +) +``` + +**Next generation prompt automatically includes:** + +```markdown +## Previous Execution Feedback +Warning: suboptimal memory access pattern +LLM Feedback: Code is correct but could use better variable names +Build Warnings: unused variable x +``` + +This creates a **feedback loop** where each generation learns from previous mistakes! + +## Visualization + +**Real-time evolution tracking** with interactive web interface: + +```bash +# Install visualization dependencies +pip install -r scripts/requirements.txt + +# Launch interactive visualizer +python scripts/visualizer.py + +# Or visualize specific checkpoint +python scripts/visualizer.py --path examples/function_minimization/openevolve_output/checkpoints/checkpoint_100/ +``` + +**Features:** + +- **Evolution tree** with parent-child relationships +- **Performance tracking** across generations +- **Code diff viewer** showing mutations +- **MAP-Elites grid** visualization +- **Multi-metric analysis** with custom dimensions + +![OpenEvolve Visualizer](../openevolve-visualizer.png) diff --git a/docs/faq.md b/docs/faq.md new file mode 100644 index 0000000000..6958147434 --- /dev/null +++ b/docs/faq.md @@ -0,0 +1,55 @@ +# FAQ + +### How much does it cost to run? + +See [Cost Estimation](llm-providers.md#cost-estimation) for detailed pricing information and cost-saving tips. + +### 🆚 How does this compare to manual optimization? + +| Aspect | Manual | OpenEvolve | +|--------|--------|------------| +| **Initial Learning** | Weeks to understand domain | Minutes to start | +| **Solution Quality** | Depends on expertise | Consistently explores novel approaches | +| **Time Investment** | Days-weeks per optimization | Hours for complete evolution | +| **Reproducibility** | Hard to replicate exact process | Perfect reproduction with seeds | +| **Scaling** | Doesn't scale beyond human capacity | Parallel evolution across islands | + +**OpenEvolve shines** when you need to explore large solution spaces or optimize for multiple objectives simultaneously. + +### Can I use my own LLM? + +**Yes!** OpenEvolve supports any OpenAI-compatible API: + +- **Commercial**: OpenAI, Google, Cohere +- **Local**: Ollama, vLLM, LM Studio, text-generation-webui +- **Advanced**: OptiLLM for routing and test-time compute + +Just set the `api_base` in your config to point to your endpoint. + +### What if evolution gets stuck? + +**Built-in mechanisms prevent stagnation:** + +- **Island migration**: Fresh genes from other populations +- **Temperature control**: Exploration vs exploitation balance +- **Diversity maintenance**: MAP-Elites prevents convergence +- **Artifact feedback**: Error messages guide improvements +- **Template stochasticity**: Randomized prompts break patterns + +**Manual interventions:** +- Increase `num_diverse_programs` for more exploration +- Add custom feature dimensions to diversify search +- Use template variations to randomize prompts +- Adjust migration intervals for more cross-pollination + +### How do I measure success? + +**Multiple success metrics:** + +1. **Primary Metric**: Your evaluator's `combined_score` or metric average +2. **Convergence**: Best score improvement over time +3. **Diversity**: MAP-Elites grid coverage +4. **Efficiency**: Iterations to reach target performance +5. **Robustness**: Performance across different test cases + +**Use the visualizer** to track all metrics in real-time and identify when evolution has converged. diff --git a/docs/how-it-works.md b/docs/how-it-works.md new file mode 100644 index 0000000000..81067bbed5 --- /dev/null +++ b/docs/how-it-works.md @@ -0,0 +1,35 @@ +# How OpenEvolve Works + +OpenEvolve implements a sophisticated **evolutionary coding pipeline** that goes far beyond simple optimization: + +![OpenEvolve Architecture](../openevolve-architecture.png) + +### **Core Innovation**: MAP-Elites + LLMs + +- **Quality-Diversity Evolution**: Maintains diverse populations across feature dimensions +- **Island-Based Architecture**: Multiple populations prevent premature convergence +- **LLM Ensemble**: Multiple models with intelligent fallback strategies +- **Artifact Side-Channel**: Error feedback improves subsequent generations + +## Features + +### Scientific Reproducibility + +- **Comprehensive Seeding**: Every component (LLM, database, evaluation) is seeded +- **Default Seed=42**: Immediate reproducible results out of the box +- **Deterministic Evolution**: Exact reproduction of runs across machines +- **Component Isolation**: Hash-based isolation prevents cross-contamination + +### Advanced LLM Integration + +- **Universal API**: Works with OpenAI, Google, Claude Code CLI, local models, and proxies +- **Intelligent Ensembles**: Weighted combinations with sophisticated fallback +- **Test-Time Compute**: Enhanced reasoning through proxy systems (see [OptiLLM setup](llm-providers.md)) +- **Plugin Ecosystem**: Support for advanced reasoning plugins + +### Evolution Algorithm Innovations + +- **Double Selection**: Different programs for performance vs inspiration +- **Adaptive Feature Dimensions**: Custom quality-diversity metrics +- **Migration Patterns**: Ring topology with controlled gene flow +- **Multi-Strategy Sampling**: Elite, diverse, and exploratory selection diff --git a/docs/llm-providers.md b/docs/llm-providers.md new file mode 100644 index 0000000000..38eb220885 --- /dev/null +++ b/docs/llm-providers.md @@ -0,0 +1,144 @@ +# LLM Providers + +OpenEvolve works with any OpenAI-compatible API, and with the Claude Code, GitHub Copilot and Codex CLIs without an API key. + +### OpenAI (Direct) + +```bash +export OPENAI_API_KEY="sk-..." +# Uses OpenAI endpoints by default +``` + +### Google Gemini + +```yaml +# config.yaml +llm: + api_base: "https://generativelanguage.googleapis.com/v1beta/openai/" + primary_model: "gemini-2.5-pro" +``` + +```bash +export OPENAI_API_KEY="your-gemini-api-key" +``` + +### Local Models (Ollama/vLLM) + +```yaml +# config.yaml +llm: + api_base: "http://localhost:11434/v1" # Ollama + primary_model: "codellama:7b" +``` + +### OptiLLM (Advanced) + +For maximum flexibility with rate limiting, model routing, and test-time compute: + +```bash +# Install OptiLLM +pip install optillm + +# Start OptiLLM proxy +optillm --port 8000 + +# Point OpenEvolve to OptiLLM +export OPENAI_API_KEY="your-actual-key" +``` + +```yaml +llm: + api_base: "http://localhost:8000/v1" + primary_model: "moa&readurls-o3" # Test-time compute + web access +``` + +### Claude Code CLI (No API Key) + +Use the [Claude Code CLI](https://docs.anthropic.com/en/docs/claude-code) as the LLM backend — no API keys needed, authentication uses the CLI's OAuth session. + +```bash +# Install and authenticate +npm install -g @anthropic-ai/claude-code +claude login +``` + +```yaml +# config.yaml +llm: + provider: "claude_code" + models: + - name: "sonnet" + weight: 0.8 + max_tokens: 16000 + max_budget_usd: 1.0 + - name: "haiku" + weight: 0.2 + max_tokens: 8000 +``` + +See the [Claude Code quickstart example](../examples/claude_code_quickstart/) for a complete walkthrough. + +### GitHub Copilot CLI (No API Key) + +Use the [GitHub Copilot CLI](https://docs.github.com/copilot/how-tos/copilot-cli) as the LLM backend — no API keys needed, authentication uses your GitHub Copilot subscription. + +```bash +# Install and authenticate +npm install -g @github/copilot +copilot login +``` + +```yaml +# config.yaml +llm: + provider: "copilot_cli" + models: + - name: "claude-sonnet-4.6" + weight: 0.8 + reasoning_effort: "medium" + - name: "gpt-5.4" + weight: 0.2 +``` + +See the [Copilot CLI quickstart example](../examples/copilot_cli_quickstart/) for a complete walkthrough. + +### Codex App Server (No API Key) + +Use the [Codex CLI's app-server](https://developers.openai.com/codex/app-server/) as the LLM backend — no API keys needed. Authentication uses your ChatGPT (Codex) subscription: the app-server owns the managed browser/device-code login and refreshes the tokens itself. OpenEvolve never reads the CLI's token files, and before each generation it verifies with the app-server that a ChatGPT-plan login is active (it fails with a clear message pointing at `codex login` otherwise). To bill an API key instead, keep the default OpenAI provider — both paths coexist. + +```bash +# Install and authenticate +npm install -g @openai/codex +codex login # browser flow, or: codex login --device-code +``` + +```yaml +# config.yaml +llm: + provider: "codex_app_server" + models: + - name: "gpt-5.3-codex" + weight: 0.8 + reasoning_effort: "medium" + - name: "gpt-5.3-codex-mini" + weight: 0.2 +``` + +Each call speaks newline-delimited JSON-RPC to a short-lived `codex app-server` process over stdio (unsupported on Windows). Threads run non-interactive: tool approvals are `never` and the sandbox is read-only, so responses are generated without mutating your workspace. + +## Cost Estimation + +**Cost depends on your LLM provider and iterations:** + +- **o3**: ~$0.15-0.60 per iteration (depending on code size) +- **o3-mini**: ~$0.03-0.12 per iteration (more cost-effective) +- **Gemini-2.5-Pro**: ~$0.08-0.30 per iteration +- **Gemini-2.5-Flash**: ~$0.01-0.05 per iteration (fastest and cheapest) +- **Local models**: Nearly free after setup +- **OptiLLM**: Use cheaper models with test-time compute for better results + +**Cost-saving tips:** +- Start with fewer iterations (100-200) +- Use o3-mini, Gemini-2.5-Flash or local models for exploration +- Use cascade evaluation to filter bad programs early +- Configure smaller population sizes initially diff --git a/docs/roadmap.md b/docs/roadmap.md new file mode 100644 index 0000000000..4c50e38438 --- /dev/null +++ b/docs/roadmap.md @@ -0,0 +1,17 @@ +# Roadmap + +### **Upcoming Features** + +- [ ] **Multi-Modal Evolution**: Images, audio, and text simultaneously +- [ ] **Federated Learning**: Distributed evolution across multiple machines +- [ ] **AutoML Integration**: Hyperparameter and architecture evolution +- [ ] **Benchmark Suite**: Standardized evaluation across domains + +### **Research Directions** + +- [ ] **Self-Modifying Prompts**: Evolution modifies its own prompting strategy +- [ ] **Cross-Language Evolution**: Python → Rust → C++ optimization chains +- [ ] **Neurosymbolic Reasoning**: Combine neural and symbolic approaches +- [ ] **Human-AI Collaboration**: Interactive evolution with human feedback + +Want to contribute? Check out our [roadmap discussions](https://github.com/algorithmicsuperintelligence/openevolve/discussions/categories/roadmap)! diff --git a/docs/showcase.md b/docs/showcase.md new file mode 100644 index 0000000000..e3f14a9331 --- /dev/null +++ b/docs/showcase.md @@ -0,0 +1,88 @@ +# Showcase + +For the full list of runnable projects, see [`examples/`](../examples/). + +## GPU Kernel Evolution + +**Before (Baseline)**: +```metal +// Standard attention implementation +kernel void attention_baseline(/* ... */) { + // Generic matrix multiplication + float sum = 0.0; + for (int i = 0; i < seq_len; i++) { + sum += query[tid] * key[i]; + } +} +``` + +**After Evolution (2.8x faster)**: +```metal +// OpenEvolve discovered optimization +kernel void attention_evolved(/* ... */) { + // Hardware-aware tiling + unified memory optimization + threadgroup float shared_mem[256]; + // ... evolved algorithm exploiting Apple Silicon architecture +} +``` + +**Performance Impact**: 2.8x speedup on Apple M1 Pro, maintaining numerical accuracy. + +## Function Minimization + +**Watch OpenEvolve evolve from random search to sophisticated optimization:** + +```python +# Initial Program (Random Search) +def minimize_function(func, bounds, max_evals=1000): + best_x, best_val = None, float('inf') + for _ in range(max_evals): + x = random_point_in_bounds(bounds) + val = func(x) + if val < best_val: + best_x, best_val = x, val + return best_x, best_val +``` + +**Evolution Process** + +```python +# Evolved Program (Simulated Annealing + Adaptive Cooling) +def minimize_function(func, bounds, max_evals=1000): + x = random_point_in_bounds(bounds) + temp = adaptive_initial_temperature(func, bounds) + + for i in range(max_evals): + neighbor = generate_neighbor(x, temp, bounds) + delta = func(neighbor) - func(x) + + if delta < 0 or random.random() < exp(-delta/temp): + x = neighbor + + temp *= adaptive_cooling_rate(i, max_evals) # Dynamic cooling + + return x, func(x) +``` + +**Performance**: 100x improvement in convergence speed! + +## Prompt Evolution + +**Evolve prompts instead of code** for better LLM performance. See the [LLM Prompt Optimization example](../examples/llm_prompt_optimization/) for a complete case study with HotpotQA achieving +23% accuracy improvement. + +[Full Example](../examples/llm_prompt_optimization/) + +## Competitive Programming + +**Automatic solution generation** for programming contests: + +```python +# Problem: Find maximum subarray sum +# OpenEvolve discovers multiple approaches: + +# Evolution Path 1: Brute Force → Kadane's Algorithm +# Evolution Path 2: Divide & Conquer → Optimized Kadane's +# Evolution Path 3: Dynamic Programming → Space-Optimized DP +``` + +[Online Judge Integration](../examples/online_judge_programming/) diff --git a/docs/system-messages.md b/docs/system-messages.md new file mode 100644 index 0000000000..969e9c33b8 --- /dev/null +++ b/docs/system-messages.md @@ -0,0 +1,151 @@ +# Crafting Effective System Messages + +**System messages are the secret to successful evolution.** They guide the LLM's understanding of your domain, constraints, and optimization goals. A well-crafted system message can be the difference between random mutations and targeted improvements. + +### Why System Messages Matter + +The system message in your config.yaml is arguably the most important component for evolution success: + +- **Domain Expertise**: Provides LLM with specific knowledge about your problem space +- **Constraint Awareness**: Defines what can and cannot be changed during evolution +- **Optimization Focus**: Guides the LLM toward meaningful improvements +- **Error Prevention**: Helps avoid common pitfalls and compilation errors + +### The Iterative Creation Process + +Based on successful OpenEvolve implementations, system messages are best created through iteration: + +### Step-by-Step Process + +**Phase 1: Initial Draft** + +1. Start with a basic system message describing your goal +2. Run 20-50 evolution iterations to observe behavior +3. Note where the system gets "stuck" or makes poor choices + +**Phase 2: Refinement** + +4. Add specific guidance based on observed issues +5. Include domain-specific terminology and concepts +6. Define clear constraints and optimization targets +7. Run another batch of iterations + +**Phase 3: Specialization** + +8. Add detailed examples of good vs bad approaches +9. Include specific library/framework guidance +10. Add error avoidance patterns you've observed +11. Fine-tune based on artifact feedback + +**Phase 4: Optimization** + +12. Consider using OpenEvolve itself to optimize your prompt +13. Measure improvements using combined score metrics + +### Examples by Complexity + +#### **Simple: General Optimization** +```yaml +prompt: + system_message: | + You are an expert programmer specializing in optimization algorithms. + Your task is to improve a function minimization algorithm to find the + global minimum reliably, escaping local minima that might trap simple algorithms. +``` + +#### **Intermediate: Domain-Specific Guidance** +```yaml +prompt: + system_message: | + You are an expert prompt engineer. Your task is to revise prompts for LLMs. + + Your improvements should: + * Clarify vague instructions and eliminate ambiguity + * Strengthen alignment between prompt and desired task outcome + * Improve robustness against edge cases + * Include formatting instructions and examples where helpful + * Avoid unnecessary verbosity + + Return only the improved prompt text without explanations. +``` + +#### **Advanced: Hardware-Specific Optimization** +```yaml +prompt: + system_message: | + You are an expert Metal GPU programmer specializing in custom attention + kernels for Apple Silicon. + + # TARGET: Optimize Metal Kernel for Grouped Query Attention (GQA) + # HARDWARE: Apple M-series GPUs with unified memory architecture + # GOAL: 5-15% performance improvement + + # OPTIMIZATION OPPORTUNITIES: + **1. Memory Access Pattern Optimization:** + - Coalesced access patterns for Apple Silicon + - Vectorized loading using SIMD + - Pre-compute frequently used indices + + **2. Algorithm Fusion:** + - Combine max finding with score computation + - Reduce number of passes through data + + # CONSTRAINTS - CRITICAL SAFETY RULES: + **MUST NOT CHANGE:** + - Kernel function signature + - Template parameter names or types + - Overall algorithm correctness + + **ALLOWED TO OPTIMIZE:** + - Memory access patterns and indexing + - Computation order and efficiency + - Vectorization and SIMD utilization + - Apple Silicon specific optimizations +``` + +### Best Practices + +### Prompt Engineering Patterns + +**Structure Your Message:** Start with role definition → Define task/context → List optimization opportunities → Set constraints → Success criteria + +**Use Specific Examples:** +```yaml +# Good: "Focus on reducing memory allocations. Example: Replace `new Vector()` with pre-allocated arrays." +# Avoid: "Make the code faster" +``` + +**Include Domain Knowledge:** +```yaml +# Good: "For GPU kernels: 1) Memory coalescing 2) Occupancy 3) Shared memory usage" +# Avoid: "Optimize the algorithm" +``` + +**Set Clear Boundaries:** +```yaml +system_message: | + MUST NOT CHANGE: - Function signatures - Algorithm correctness - External API + ALLOWED: - Internal implementation - Data structures - Performance optimizations +``` + +### Advanced Techniques + +**Artifact-Driven Iteration:** Enable artifacts in config → Include common error patterns in system message → Add guidance based on stderr/warning patterns + +**Multi-Phase Evolution:** Start broad ("Explore different algorithmic approaches"), then focus ("Given successful simulated annealing, focus on parameter tuning") + +**Template Stochasticity:** See the [Configuration section](configuration.md) for complete template variation examples. + +### Meta-Evolution: Using OpenEvolve to Optimize Prompts + +**You can use OpenEvolve to evolve your system messages themselves!** This powerful technique lets you optimize prompts for better LLM performance automatically. + +See the [LLM Prompt Optimization example](../examples/llm_prompt_optimization/) for a complete implementation, including the HotpotQA case study with +23% accuracy improvement. + +### Common Pitfalls to Avoid + +- **Too Vague**: "Make the code better" → Specify exactly what "better" means +- **Too Restrictive**: Over-constraining can prevent useful optimizations +- **Missing Context**: Include relevant domain knowledge and terminology +- **No Examples**: Concrete examples guide LLM better than abstract descriptions +- **Ignoring Artifacts**: Don't refine prompts based on error feedback