diff --git a/openevolve/evolution_trace.py b/openevolve/evolution_trace.py index 8e6699cda..957bc1020 100644 --- a/openevolve/evolution_trace.py +++ b/openevolve/evolution_trace.py @@ -8,6 +8,7 @@ import json import logging import time +from copy import deepcopy from dataclasses import asdict, dataclass from pathlib import Path from typing import Any, Dict, List, Optional, Union @@ -164,12 +165,12 @@ def log_trace( timestamp=time.time(), parent_id=parent_program.id, child_id=child_program.id, - parent_metrics=parent_program.metrics, - child_metrics=child_program.metrics, + parent_metrics=deepcopy(parent_program.metrics), + child_metrics=deepcopy(child_program.metrics), island_id=island_id, generation=child_program.generation, - artifacts=artifacts, - metadata=metadata, + artifacts=deepcopy(artifacts), + metadata=deepcopy(metadata), ) # Optionally include code @@ -179,12 +180,16 @@ def log_trace( # Changes descriptions (large-codebase mode) if self.include_changes_description: - trace.parent_changes_description = getattr(parent_program, "changes_description", None) - trace.child_changes_description = getattr(child_program, "changes_description", None) + trace.parent_changes_description = getattr( + parent_program, "changes_description", None + ) + trace.child_changes_description = getattr( + child_program, "changes_description", None + ) # Optionally include prompts if self.include_prompts: - trace.prompt = prompt + trace.prompt = deepcopy(prompt) trace.llm_response = llm_response # Calculate improvement @@ -294,7 +299,9 @@ def close(self): "created_at": time.time(), "include_code": self.include_code, "include_prompts": self.include_prompts, - "include_changes_description": getattr(self, "include_changes_description", True), + "include_changes_description": getattr( + self, "include_changes_description", True + ), } export_traces(all_traces, self.output_path, format="hdf5", metadata=metadata) diff --git a/tests/test_evolution_trace_snapshots.py b/tests/test_evolution_trace_snapshots.py new file mode 100644 index 000000000..882b9bff0 --- /dev/null +++ b/tests/test_evolution_trace_snapshots.py @@ -0,0 +1,88 @@ +"""Regression tests for evolution traces recorded from reused mutable payloads.""" + +import json +import tempfile +import unittest +from pathlib import Path + +from openevolve.database import Program +from openevolve.evolution_trace import EvolutionTracer + + +class TestEvolutionTraceSnapshots(unittest.TestCase): + def test_reused_program_metrics_preserve_each_transition(self): + for format, buffer_size in (("jsonl", 1), ("jsonl", 10), ("json", 1), ("json", 10)): + with ( + self.subTest(format=format, buffer_size=buffer_size), + tempfile.TemporaryDirectory() as directory, + ): + path = Path(directory) / f"trace.{format}" + parent = Program(id="parent", code="pass", metrics={"combined_score": 1.0}) + child = Program(id="child", code="pass", metrics={"combined_score": 3.0}) + with EvolutionTracer(str(path), format=format, buffer_size=buffer_size) as tracer: + tracer.log_trace(1, parent, child) + parent.metrics["combined_score"] = 10.0 + child.metrics["combined_score"] = 30.0 + tracer.log_trace(2, parent, child) + parent.metrics.clear() + child.metrics["combined_score"] = 100.0 + + if format == "jsonl": + traces = [json.loads(line) for line in path.read_text().splitlines()] + else: + traces = json.loads(path.read_text())["traces"] + self.assertEqual(len(traces), 2) + for trace, parent_score, child_score in zip(traces, (1.0, 10.0), (3.0, 30.0)): + self.assertEqual(trace["parent_metrics"], {"combined_score": parent_score}) + self.assertEqual(trace["child_metrics"], {"combined_score": child_score}) + self.assertEqual( + trace["improvement_delta"], {"combined_score": child_score - parent_score} + ) + self.assertEqual( + tracer.get_statistics()["total_improvement"], {"combined_score": 22.0} + ) + + def test_prompt_artifacts_and_metadata_are_deep_snapshots(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "trace.jsonl" + parent = Program(id="parent", code="pass") + child = Program(id="child", code="pass") + prompt = {"system": "original"} + artifacts = {"evaluation": {"messages": ["original"]}} + metadata = {"worker": {"attempts": [1]}} + with EvolutionTracer(str(path), buffer_size=10) as tracer: + tracer.log_trace( + 1, parent, child, prompt=prompt, artifacts=artifacts, metadata=metadata + ) + prompt["system"] = "changed" + artifacts["evaluation"]["messages"].append("changed") + metadata["worker"]["attempts"].append(2) + tracer.log_trace( + 2, parent, child, prompt=prompt, artifacts=artifacts, metadata=metadata + ) + prompt.clear() + artifacts.clear() + metadata.clear() + + first, second = [json.loads(line) for line in path.read_text().splitlines()] + self.assertEqual(first["prompt"], {"system": "original"}) + self.assertEqual(first["artifacts"], {"evaluation": {"messages": ["original"]}}) + self.assertEqual(first["metadata"], {"worker": {"attempts": [1]}}) + self.assertEqual(second["prompt"], {"system": "changed"}) + self.assertEqual( + second["artifacts"], {"evaluation": {"messages": ["original", "changed"]}} + ) + self.assertEqual(second["metadata"], {"worker": {"attempts": [1, 2]}}) + + def test_excluded_prompts_stay_out_of_trace(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "trace.jsonl" + parent = Program(id="parent", code="pass") + child = Program(id="child", code="pass") + with EvolutionTracer(str(path), include_prompts=False) as tracer: + tracer.log_trace( + 1, parent, child, prompt={"system": "private"}, llm_response="private" + ) + trace = json.loads(path.read_text()) + self.assertNotIn("prompt", trace) + self.assertNotIn("llm_response", trace)