diff --git a/openevolve/_version.py b/openevolve/_version.py index 57b6d4eb9..ddad2c119 100644 --- a/openevolve/_version.py +++ b/openevolve/_version.py @@ -1,3 +1,3 @@ """Version information for openevolve package.""" -__version__ = "0.2.20" +__version__ = "0.2.21" diff --git a/openevolve/database.py b/openevolve/database.py index 1c7f73937..f7b10bdb2 100644 --- a/openevolve/database.py +++ b/openevolve/database.py @@ -975,20 +975,36 @@ def _llm_judge_novelty(self, program: Program, similar_program: Program) -> bool """ import asyncio from openevolve.novelty_judge import NOVELTY_SYSTEM_MSG, NOVELTY_USER_MSG - + user_msg = NOVELTY_USER_MSG.format( language=program.language, existing_code=similar_program.code, proposed_code=program.code, ) - + try: - content: str = asyncio.run( - self.novelty_llm.generate_with_context( - system_message=NOVELTY_SYSTEM_MSG, - messages=[{"role": "user", "content": user_msg}], + # Check if we're already in an event loop + try: + loop = asyncio.get_running_loop() + # We're in an async context, need to run in a new thread + import concurrent.futures + with concurrent.futures.ThreadPoolExecutor() as executor: + future = executor.submit( + asyncio.run, + self.novelty_llm.generate_with_context( + system_message=NOVELTY_SYSTEM_MSG, + messages=[{"role": "user", "content": user_msg}], + ) + ) + content: str = future.result() + except RuntimeError: + # No event loop running, safe to use asyncio.run() + content: str = asyncio.run( + self.novelty_llm.generate_with_context( + system_message=NOVELTY_SYSTEM_MSG, + messages=[{"role": "user", "content": user_msg}], + ) ) - ) if content is None or content is None: logger.warning("Novelty LLM returned empty response") @@ -999,11 +1015,11 @@ def _llm_judge_novelty(self, program: Program, similar_program: Program) -> bool # Parse the response NOVEL_i = content.upper().find("NOVEL") NOT_NOVEL_i = content.upper().find("NOT NOVEL") - + if NOVEL_i == -1 and NOT_NOVEL_i == -1: logger.warning(f"Unexpected novelty LLM response: {content}") return True # Assume novel if we can't parse - + if NOVEL_i != -1 and NOT_NOVEL_i != -1: # Both found, take the one that appears first is_novel = NOVEL_i < NOT_NOVEL_i @@ -1011,12 +1027,12 @@ def _llm_judge_novelty(self, program: Program, similar_program: Program) -> bool is_novel = True else: is_novel = False - + return is_novel except Exception as e: logger.error(f"Error in novelty LLM check: {e}") - + return True def _is_novel(self, program_id: int, island_idx: int) -> bool: diff --git a/tests/test_novelty_asyncio_issue.py b/tests/test_novelty_asyncio_issue.py new file mode 100644 index 000000000..46fb03475 --- /dev/null +++ b/tests/test_novelty_asyncio_issue.py @@ -0,0 +1,169 @@ +""" +Test for issue #313: asyncio.run() error in novelty checking +https://github.com/algorithmicsuperintelligence/openevolve/issues/313 + +This test reproduces the bug where calling database.add() from within an async context +triggers a novelty check that uses asyncio.run(), which fails because it's already +running in an event loop. +""" + +import unittest +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch, Mock +from openevolve.config import Config +from openevolve.database import Program, ProgramDatabase + + +class MockLLM: + """Mock LLM that implements the async interface""" + + async def generate_with_context(self, system_message: str, messages: list): + """Mock async generate method that returns NOVEL""" + return "NOVEL" + + +class TestNoveltyAsyncioIssue(unittest.TestCase): + """Test for asyncio.run() error in novelty checking (issue #313)""" + + @patch('openevolve.embedding.EmbeddingClient') + def setUp(self, mock_embedding_client_class): + """Set up test database with novelty checking enabled""" + # Mock the embedding client + mock_instance = MagicMock() + mock_instance.get_embedding.return_value = [0.1] * 1536 # Mock embedding vector + mock_embedding_client_class.return_value = mock_instance + + config = Config() + config.database.in_memory = True + config.database.embedding_model = "text-embedding-3-small" + config.database.similarity_threshold = 0.99 + config.database.novelty_llm = MockLLM() + + self.db = ProgramDatabase(config.database) + self.mock_embedding_client_class = mock_embedding_client_class + + def test_novelty_check_from_async_context_works(self): + """ + Test that novelty checking works correctly when called from within + an async context (this was the bug in issue #313). + + Expected behavior: Should successfully run the novelty check without + any asyncio.run() errors, properly using ThreadPoolExecutor to handle + the async LLM call from within a running event loop. + """ + import logging + + # Create two programs with similar embeddings to trigger LLM novelty check + program1 = Program( + id="prog1", + code="def test(): return 1", + language="python", + metrics={"score": 0.5}, + ) + + program2 = Program( + id="prog2", + code="def test(): return 2", + language="python", + metrics={"score": 0.6}, + parent_id="prog1", + ) + + async def async_add_programs(): + """Add programs from async context - this simulates controller.run()""" + # Add first program (no novelty check, no similar programs yet) + prog1_id = self.db.add(program1) + self.assertIsNotNone(prog1_id) + + # Add second program - this triggers novelty check + # Since embeddings are similar (both [0.1] * 1536), it will call + # _llm_judge_novelty which should now work correctly + prog2_id = self.db.add(program2) + + # The novelty check should succeed without errors + # The program should be added (MockLLM returns "NOVEL") + self.assertIsNotNone(prog2_id) + + return True + + # This should work without any errors now + result = asyncio.run(async_add_programs()) + self.assertTrue(result) + + # Verify both programs were added + self.assertIn("prog1", self.db.programs) + self.assertIn("prog2", self.db.programs) + + def test_novelty_check_from_sync_context_works(self): + """ + Test that novelty checking also works correctly when called from + a synchronous (non-async) context. + + Expected behavior: Should successfully run the novelty check using + asyncio.run() since there's no running event loop. + """ + # Create two programs with similar embeddings to trigger LLM novelty check + program1 = Program( + id="prog3", + code="def test(): return 3", + language="python", + metrics={"score": 0.5}, + ) + + program2 = Program( + id="prog4", + code="def test(): return 4", + language="python", + metrics={"score": 0.6}, + parent_id="prog3", + ) + + # Add programs from synchronous context (no event loop running) + prog1_id = self.db.add(program1) + self.assertIsNotNone(prog1_id) + + prog2_id = self.db.add(program2) + self.assertIsNotNone(prog2_id) + + # Verify both programs were added + self.assertIn("prog3", self.db.programs) + self.assertIn("prog4", self.db.programs) + + def test_novelty_check_disabled_works_fine(self): + """ + Test that when novelty checking is disabled, adding programs + from async context works fine (this is the workaround from issue #313). + """ + # Create a new database with novelty checking disabled + config = Config() + config.database.in_memory = True + config.database.similarity_threshold = 0.0 # Disable novelty checking + db_no_novelty = ProgramDatabase(config.database) + + program1 = Program( + id="prog1", + code="def test(): return 1", + language="python", + metrics={"score": 0.5}, + ) + + program2 = Program( + id="prog2", + code="def test(): return 2", + language="python", + metrics={"score": 0.6}, + ) + + async def async_add_programs(): + """Add programs from async context""" + db_no_novelty.add(program1) + db_no_novelty.add(program2) + return True + + # This should work fine without novelty checking + result = asyncio.run(async_add_programs()) + self.assertTrue(result) + + +if __name__ == "__main__": + unittest.main()