From f47ee108a994b4755208671809fb6df60d3ee145 Mon Sep 17 00:00:00 2001 From: alalulu8668 Date: Mon, 30 Oct 2023 05:28:32 +0100 Subject: [PATCH 1/5] add memory to microscopist --- .../teams/image_analysis_hub/microscopist.py | 89 ++++++++++++++----- .../teams/image_analysis_hub/schemas.py | 9 +- 2 files changed, 70 insertions(+), 28 deletions(-) diff --git a/schema_agents/teams/image_analysis_hub/microscopist.py b/schema_agents/teams/image_analysis_hub/microscopist.py index 330c470..8e7099f 100644 --- a/schema_agents/teams/image_analysis_hub/microscopist.py +++ b/schema_agents/teams/image_analysis_hub/microscopist.py @@ -3,10 +3,10 @@ from pydantic import BaseModel, Field from schema_agents.role import Role -from schema_agents.schema import Message -from schema_agents.tools.code_interpreter import create_mock_client +from schema_agents.schema import Message, MemoryChunk from schema_agents.memory.long_term_memory import LongTermMemory - +from schema_agents.tools.code_interpreter import create_mock_client +from schemas import (FunctionMemory, ExperienceMemory) class MicroscopeControlRequirements(BaseModel): """Requirements for controlling the microscope and acquire images.""" @@ -14,13 +14,10 @@ class MicroscopeControlRequirements(BaseModel): timeout: float = Field(default=0.0, description="timeout") query: str = Field(default="", description="user's original request") plan: str = Field(default="", description="plan for control microscope and acquiring images") + class MultiDimensionalAcquisitionScript(BaseModel): - """Python script for simple and complex multi-dimensional acquisition. - In the script, you can use the following functions to control the microscope: - - `microscope_move({'x': 0.0, 'y': 0.0, 'z': 0.0})` # x, y, z are in microns - - `microscope_snap({'path': './images', 'exposure': 0.0})` # path is the path to save the image, exposure is in seconds - """ + """Python script for simple and complex multi-dimensional acquisition. Use the provided 'Experiences' to generate the script.""" script: str = Field(default="", description="Script for acquiring multi-dimensional images") explanation: str = Field(default="", description="Brief explanation for the script") timeout: float = Field(default=0.0, description="a reasonable timeout for executing the script") @@ -31,19 +28,47 @@ class ExecutionResult(BaseModel): outputs: List[Dict[str, Any]] = Field(default=[], description="Outputs of executing the script") traceback: Optional[str] = Field(default=None, description="Traceback of executing the script") -INIT_SCRIPT = """ -def microscope_move(position): - print(f"===> Moving to: {position}") -def microscope_snap(config): - print(f"===> Snapped an image with exposure {config['exposure']} and saved to: { config['path']}") -""" +class FunctionDescription(BaseModel): + """Description of a function.""" + name: str = Field(description="Name of the function") + docstring: str = Field(description="Docstring of the function") + args: List[str] = Field(default=[], description="Arguments of the function with type annotation") + + +class ScriptGenerationContext(BaseModel): + """The context for generating the script.""" + registed_functions: List[FunctionDescription] = Field(default=[], description="A list of registed functions which can be used directly in the script.") + experience_memory: List[ExperienceMemory] = Field(default=[], description="A list of experience memories to help create the script and avoid common mistakes.") + +def create_long_term_memory(): + memory = LongTermMemory() + role_id = 'bio' + memory.recover_memory(role_id) + memory.clean() + + function_move = FunctionMemory(function_name='microscope_move', code="""def microscope_move(position): + print(f"===> Moving to: {position}")""", lang='python', args=['position:Tuple[float,float,float]'], docstring='Move the microscope to the given position.') + function_snap = FunctionMemory(function_name='microscope_snap', code="""def microscope_snap(config): + print(f"===> Snapped an image with exposure {config['exposure']} and saved to: { config['path']}")""", lang='python', args=['config:Dict[str,Any]'], + docstring='Snap an image with the given configuration. The input config should contain the exposure time (key=exposure) and the path (key=path) to save the image.') + + exp = ExperienceMemory(summary='Microscope move restriction', keypoints='Make sure each movement on microscope is larger than 5nm.') + + exp_memo = MemoryChunk(index='Experience for running microscope_move function', content=exp, category='experience') + memory.add(exp_memo) + fun1_memo = MemoryChunk(index='microscope move python function',content=function_move, category='function') + memory.add(fun1_memo) + fun2_memo = MemoryChunk(index='microscope snap python function',content=function_snap, category='function') + memory.add(fun2_memo) + memories = memory.recover_memory(role_id) + return memory class Microscope(): def __init__(self, client): self.client = client - self.initialized = False + self.initialized = True async def plan(self, query: str=None, role: Role=None) -> MicroscopeControlRequirements: """Make a plan for image acquisition tasks.""" @@ -51,42 +76,60 @@ async def plan(self, query: str=None, role: Role=None) -> MicroscopeControlRequi async def multi_dimensional_acquisition(self, config: MicroscopeControlRequirements=None, role: Role=None) -> ExecutionResult: """Perform image acquisition by using Python script.""" - if not self.initialized: - await self.client.executeScript({"script": INIT_SCRIPT}) - self.initialized = True + + function_memories = role.long_term_memory.retrieve("microscope related functions", filter={"category": "function"}) + function_list = [] + for memory in function_memories: + script = memory.content.code + function_list.append(FunctionDescription(name=memory.content.function_name, docstring=memory.content.docstring, args=memory.content.args)) + await self.client.executeScript({"script": script}) + experiences = role.long_term_memory.retrieve("microscope related function", filter={"category": "experience"}) + exp_list = [] + for exp in experiences: + exp_list.append(exp.content) + context = ScriptGenerationContext(registed_functions=function_list, experience_memory=exp_list) print("Acquiring images in multiple dimensions: " + str(config)) - controlScript = await role.aask(config, MultiDimensionalAcquisitionScript) + + inputs = [context, config, """Make sure each movement on the microscope stage is larger than 5nm."""] + controlScript = await role.aask(inputs, MultiDimensionalAcquisitionScript) result = await self.client.executeScript({"script": controlScript.script, "timeout": controlScript.timeout}) + return ExecutionResult( status=result['status'], outputs=result['outputs'], traceback=result.get("traceback") ) -def create_microscopist(client=None): +def create_microscopist_with_ltm(client=None): if not client: client = create_mock_client() + microscope = Microscope(client) Microscopist = Role.create( name="Thomas", profile="Microscopist", goal="Acquire images from the microscope based on user's requests.", constraints=None, - actions=[microscope.multi_dimensional_acquisition], + actions=[microscope.plan, microscope.multi_dimensional_acquisition], ) + return Microscopist + async def main(): client = create_mock_client() microscope = Microscope(client) + Microscopist = Role.create( name="Thomas", profile="Microscopist", goal="Acquire images from the microscope based on user's requests.", constraints=None, actions=[microscope.plan, microscope.multi_dimensional_acquisition], + long_term_memory=create_long_term_memory(), ) ms = Microscopist() + ms.recv(Message(content="acquire image every 2nm along x, y in a 2x2um square, gradually increase exposure time from 0.1 to 2.0s", role="User")) resp = await ms._react() print(resp) @@ -94,6 +137,7 @@ async def main(): ms.recv(res) resp = await ms._react() print(resp) + ms.recv(Message(content="acquire an image and save to /tmp/img.png", role="User")) resp = await ms._react() @@ -111,7 +155,10 @@ async def main(): resp = await ms._react() print(resp) + ms.long_term_memory.clean() + assert ms.long_term_memory.is_initialized is False if __name__ == "__main__": asyncio.run(main()) + # create_memory_storage() diff --git a/schema_agents/teams/image_analysis_hub/schemas.py b/schema_agents/teams/image_analysis_hub/schemas.py index 7666e9a..08e9382 100644 --- a/schema_agents/teams/image_analysis_hub/schemas.py +++ b/schema_agents/teams/image_analysis_hub/schemas.py @@ -45,13 +45,8 @@ class FunctionMemory(BaseModel): function_name: str = Field(default="", description="Function name") code: str = Field(default="", description="original code of the function") lang: str = Field(default="", description="function language") - args: List[str] = Field(default=[], description="arguments of the function") - -class ErrorMemory(BaseModel): - """Experience of making errors to be saved in the long term memory.""" - error: str = Field(default="", description="Error description") - cause_by: str = Field(default="", description="Cause of the error") - solution: str = Field(default="", description="Solution to fix the error") + args: List[str] = Field(default=[], description="arguments of the function, with type annotation") + docstring: Optional[str] = Field(default=None, description="docstring of the function") class ExperienceMemory(BaseModel): """Experience to be saved in the long term memory.""" From 6a9fdeb0ac63e20dc5c1d59ee7905fcce4d10ef6 Mon Sep 17 00:00:00 2001 From: alalulu8668 Date: Mon, 30 Oct 2023 05:29:04 +0100 Subject: [PATCH 2/5] delete file --- .../image_analysis_hub/microscopist_w_ltm.py | 155 ------------------ 1 file changed, 155 deletions(-) delete mode 100644 schema_agents/teams/image_analysis_hub/microscopist_w_ltm.py diff --git a/schema_agents/teams/image_analysis_hub/microscopist_w_ltm.py b/schema_agents/teams/image_analysis_hub/microscopist_w_ltm.py deleted file mode 100644 index 3810d38..0000000 --- a/schema_agents/teams/image_analysis_hub/microscopist_w_ltm.py +++ /dev/null @@ -1,155 +0,0 @@ -import asyncio -from typing import Any, Dict, List, Optional, Union - -from pydantic import BaseModel, Field -from schema_agents.role import Role -from schema_agents.schema import Message, MemoryChunk -from schema_agents.memory.long_term_memory import LongTermMemory -from schema_agents.tools.code_interpreter import create_mock_client -from .schemas import (FunctionMemory, ErrorMemory) - -class MicroscopeControlRequirements(BaseModel): - """Requirements for controlling the microscope and acquire images.""" - path: str = Field(default="", description="save images path") - timeout: float = Field(default=0.0, description="timeout") - query: str = Field(default="", description="user's original request") - plan: str = Field(default="", description="plan for control microscope and acquiring images") - experiences: List[MemoryChunk] = Field(default=[], description="experiences of making errors") - -class MultiDimensionalAcquisitionScript(BaseModel): - """Python script for simple and complex multi-dimensional acquisition. - In the script, you can use the following functions to control the microscope: - - `microscope_move({'x': 0.0, 'y': 0.0, 'z': 0.0})` # x, y, z are in microns - - `microscope_snap({'path': './images', 'exposure': 0.0})` # path is the path to save the image, exposure is in seconds - """ - script: str = Field(default="", description="Script for acquiring multi-dimensional images") - explanation: str = Field(default="", description="Brief explanation for the script") - timeout: float = Field(default=0.0, description="a reasonable timeout for executing the script") - -class ExecutionResult(BaseModel): - """Result of executing a Python script.""" - status: str = Field(description="Status of executing the script") - outputs: List[Dict[str, Any]] = Field(default=[], description="Outputs of executing the script") - traceback: Optional[str] = Field(default=None, description="Traceback of executing the script") - - - -def create_long_term_memory(): - memory = LongTermMemory() - role_id = 'bio' - memory.recover_memory(role_id) - memory.clean() - - function_move = FunctionMemory(function_name='microscope_move', code="""def microscope_move(position): - print(f"===> Moving to: {position}")""", lang='python', args=['position']) - function_snap = FunctionMemory(function_name='microscope_snap', code="""def microscope_snap(config): - print(f"===> Snapped an image with exposure {config['exposure']} and saved to: { config['path']}")""", lang='python', args=['config']) - - error = ErrorMemory(error='Microscope move can not be obtained for less than 5nm', solution='Make sure each movement is larger than 5nm') - - error_memo = MemoryChunk(index='Error made for microscope_move function', content=error, category='error') - memory.add(error_memo) - new_memory = MemoryChunk(index='microscope move python function',content=function_move, category='function') - memory.add(new_memory) - new_memory = MemoryChunk(index='microscope snap python function',content=function_snap, category='function') - memory.add(new_memory) - memories = memory.recover_memory(role_id) - return memory - - -class Microscope(): - def __init__(self, client): - self.client = client - self.initialized = False - - async def plan(self, query: str=None, role: Role=None) -> MicroscopeControlRequirements: - """Make a plan for image acquisition tasks.""" - return await role.aask(query, MicroscopeControlRequirements) - - async def multi_dimensional_acquisition(self, config: MicroscopeControlRequirements=None, role: Role=None) -> ExecutionResult: - """Perform image acquisition by using Python script.""" - if not self.initialized: - memories = role.long_term_memory.retrieve("microscope related functions", filter={"category": "function"}) - for memory in memories: - script = memory.content.code - await self.client.executeScript({"script": script}) - self.initialized = True - - experiences = role.long_term_memory.retrieve("microscope related function", filter={"category": "error"}) - config.experiences = experiences - print("Acquiring images in multiple dimensions: " + str(config)) - controlScript = await role.aask(config, MultiDimensionalAcquisitionScript) - result = await self.client.executeScript({"script": controlScript.script, "timeout": controlScript.timeout}) - if result['status'] != 'ok': - new_experience = await role.aask('summarize the error experience', ErrorMemory) - error_memo = MemoryChunk(index='Error made for microscope_move function', content=new_experience, category='error') - role.long_term_memory.add(error_memo) - - return ExecutionResult( - status=result['status'], - outputs=result['outputs'], - traceback=result.get("traceback") - ) - -def create_microscopist_with_ltm(client=None): - if not client: - client = create_mock_client() - - microscope = Microscope(client) - Microscopist = Role.create( - name="Thomas", - profile="Microscopist", - goal="Acquire images from the microscope based on user's requests.", - constraints=None, - actions=[microscope.plan, microscope.multi_dimensional_acquisition], - ) - - return Microscopist - - -async def main(): - client = create_mock_client() - microscope = Microscope(client) - - Microscopist = Role.create( - name="Thomas", - profile="Microscopist", - goal="Acquire images from the microscope based on user's requests.", - constraints=None, - actions=[microscope.plan, microscope.multi_dimensional_acquisition], - long_term_memory=create_long_term_memory(), - ) - ms = Microscopist() - - ms.recv(Message(content="acquire image every 2nm along x, y in a 2x2um square, gradually increase exposure time from 0.1 to 2.0s", role="User")) - resp = await ms._react() - print(resp) - for res in resp: - ms.recv(res) - resp = await ms._react() - print(resp) - - - ms.recv(Message(content="acquire an image and save to /tmp/img.png", role="User")) - resp = await ms._react() - print(resp) - for res in resp: - ms.recv(res) - resp = await ms._react() - print(resp) - - ms.recv(Message(content="acquire an image every 1 second for 10 seconds", role="User")) - resp = await ms._react() - print(resp) - for res in resp: - ms.recv(res) - resp = await ms._react() - print(resp) - - ms.long_term_memory.clean() - assert ms.long_term_memory.is_initialized is False - -if __name__ == "__main__": - asyncio.run(main()) - # create_memory_storage() - From b6b7e9481f71c0211a5d772ded3bfb3e19bdccbc Mon Sep 17 00:00:00 2001 From: alalulu8668 Date: Tue, 31 Oct 2023 02:36:14 +0100 Subject: [PATCH 3/5] add memory to data engineer --- .../teams/image_analysis_hub/data_engineer.py | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/schema_agents/teams/image_analysis_hub/data_engineer.py b/schema_agents/teams/image_analysis_hub/data_engineer.py index c8822a8..aff73f4 100644 --- a/schema_agents/teams/image_analysis_hub/data_engineer.py +++ b/schema_agents/teams/image_analysis_hub/data_engineer.py @@ -4,10 +4,11 @@ from typing import List, Union from schema_agents.role import Role -from schema_agents.schema import Message +from schema_agents.schema import Message, MemoryChunk +from schema_agents.memory.long_term_memory import LongTermMemory from schema_agents.tools.code_interpreter import create_mock_client -from .schemas import (PythonFunctionScript, PythonFunctionScriptChanges, Change, +from schema_agents.teams.image_analysis_hub.schemas import (PythonFunctionScript, PythonFunctionScriptChanges, Change, PythonFunctionScriptWithLineNumber, SoftwareRequirement) DEPLOY_SCRIPT = """ @@ -83,7 +84,8 @@ async def fix_code( ) response = await role.aask( python_function_with_line_number, - Union[PythonFunctionScriptChanges, PythonFunctionScript], + # Union[PythonFunctionScriptChanges, PythonFunctionScript], + PythonFunctionScriptChanges, prompt=prompt, ) if isinstance(response, PythonFunctionScript): @@ -117,6 +119,12 @@ async def fix_code( ) return await fix_code(role, client, python_function, output_summary) + # add the memory of experience + experience = response.experience + if experience: + error_memo = MemoryChunk(index=experience.summary, content=experience, category='error') + role.long_term_memory.add(error_memo) + return PythonFunctionScript( function_names=python_function.function_names, docstring=python_function.docstring, @@ -129,6 +137,13 @@ async def fix_code( async def generate_code( req: SoftwareRequirement, role: Role ) -> PythonFunctionScript: + # retrieve req-related memories from long term memory + memories = role.long_term_memory.retrieve(req.original_requirements, filter={"category": "error"}) + if memories: + # concatenate the error experiences + error = "\n".join([m.content.error for m in memories]) + req.additional_notes += f"\nPlease avoid the following error: {error}" # TODO, what to retreive from the error memory? + return await role.aask(req, PythonFunctionScript) INSTALL_SCRIPT = """ @@ -174,11 +189,20 @@ async def test_run_python_function( ) return python_function +def create_long_term_memory(): + memory = LongTermMemory() + role_id = 'data_engineer' + memory.recover_memory(role_id) + memory.clean() # this is for test purpose, to avoid the memory to be accumulated + + return memory + def create_data_engineer(client=None): async def develop_python_functions( req: SoftwareRequirement, role: Role ) -> PythonFunctionScript: """Develop python functions based on software requirements.""" + # if isinstance(req, SoftwareRequirement): func = await generate_code(req, role) try: @@ -187,6 +211,7 @@ async def develop_python_functions( req.additional_notes += f"\nPlease avoid the following error: {exp}" func = await generate_code(req, role) func = await test_run_python_function(role, client, req.id, func) + return func # else: # if client: @@ -204,6 +229,7 @@ async def develop_python_functions( goal="Develop the python function script according to the software requirement, ensuring that it fulfills the desired functionality. Implement necessary algorithms, handle data processing, and write tests to validate the correctness of the function.", constraints=None, actions=[develop_python_functions], + long_term_memory=create_long_term_memory(), ) return DataEngineer @@ -238,7 +264,7 @@ async def main(): "additional_notes": "The cells are U2OS cells in a IF microscopy image. The cells are round and in green color, the background is black. The number of cells in the image should be more than 12.", } - DataEngineer = create_data_engineer("test-service", client=create_mock_client()) + DataEngineer = create_data_engineer(client=create_mock_client()) ds = DataEngineer() pr = SoftwareRequirement.parse_obj(mock_software_requirements) From ee4381d1fb9447a273780a977a82adf753ad242f Mon Sep 17 00:00:00 2001 From: alalulu8668 Date: Mon, 13 Nov 2023 02:16:10 +0100 Subject: [PATCH 4/5] Merge branch 'main' of github.com:aicell-lab/schema-agents into add-memory --- tests/test_HumanEval.py | 115 +++++++++++++++++++++++++++------------- 1 file changed, 77 insertions(+), 38 deletions(-) diff --git a/tests/test_HumanEval.py b/tests/test_HumanEval.py index b32aa9b..6bf6839 100644 --- a/tests/test_HumanEval.py +++ b/tests/test_HumanEval.py @@ -1,12 +1,17 @@ -from schema_agents.role import Role + import random import os import json from typing import List, Union, Optional, Dict, Any + from pydantic import BaseModel, Field +from human_eval.evaluation import evaluate_functional_correctness from human_eval.data import write_jsonl, read_problems -from schema_agents.teams.image_analysis_hub.schemas import PythonFunctionScript +from schema_agents.tools.code_interpreter import create_mock_client +from schema_agents.schema import Message +from schema_agents.teams.image_analysis_hub.schemas import PythonFunctionScript +from schema_agents.role import Role class PythonOutput(BaseModel): """Represents a Python function with all its properties.""" @@ -15,12 +20,7 @@ class PythonOutput(BaseModel): docstring: Optional[str] = Field(None, description="Brief notes for usage, debugging, potential error fixing, and further improvements.") -async def generate_code( - req, role: Role -) -> PythonOutput: - # retrieve req-related memories from long term memory - response = await role.aask(req, PythonOutput) - return response + # async def test_run_python_function( # role, client, service_id, python_function: PythonFunctionScript @@ -46,24 +46,54 @@ async def generate_code( # return python_function -def create_data_engineer(client=None): - async def develop_python_functions( - req, role: Role - ) -> PythonOutput: - """Complete python functions based on CodeEvalInput to pass the tests.""" - - # if isinstance(req, SoftwareRequirement): - func = await generate_code(req, role) - # try: - # func = await test_run_python_function(role, client, req.id, func) - # except RuntimeError as exp: - # req.additional_notes += f"\nPlease avoid the following error: {exp}" - # func = await generate_code(req, role) - # func = await test_run_python_function(role, client, req.id, func) - return func +async def generate_code( + req, role: Role +) -> PythonOutput: + """Complete python functions based on the given input.""" + # retrieve req-related memories from long term memory + response = await role.aask(req, PythonOutput) + return response + +async def develop_python_functions( + req: str, role: Role +) -> PythonOutput: + """Complete python functions based on CodeEvalInput to pass the tests.""" + + # if isinstance(req, SoftwareRequirement): + func = await generate_code(req, role) + # try: + # func = await test_run_python_function(role, client, req.id, func) + # except RuntimeError as exp: + # req.additional_notes += f"\nPlease avoid the following error: {exp}" + # func = await generate_code(req, role) + # func = await test_run_python_function(role, client, req.id, func) + + return func + +async def main(): + # Your existing code... + + problems = read_problems() + test_path = "/home/alalulu/workspace/schema-agents/tests/data" + + selected_problems = dict(random.sample(problems.items(), 20)) + write_jsonl(os.path.join(test_path, "selected_problems.jsonl.gz"), selected_problems.values()) + + sub_problem = read_problems(os.path.join(test_path, "selected_problems.jsonl.gz")) + + num_samples_per_task = 1 + samples = await asyncio.gather(*[ + generate_sample(task_id, sub_problem[task_id]["prompt"], num_samples_per_task) + for task_id in sub_problem + ]) + + write_jsonl(os.path.join(test_path, "samples.jsonl"), samples) + await evaluate_functional_correctness("/home/alalulu/workspace/schema-agents/tests/data/samples.jsonl", [1], problem_file=os.path.join(test_path, "selected_problems.jsonl.gz")) + +async def generate_sample(task_id, prompt, num_samples): data_engineer = Role( name="Alice", profile="Data Engineer", @@ -71,18 +101,27 @@ async def develop_python_functions( constraints=None, actions=[develop_python_functions], ) - return data_engineer - -problems = read_problems() -DataEngineer = create_data_engineer() -ds = DataEngineer() -# get the first sample in problems -sub_problem = dict(random.sample(problems.items(), 1)) -num_samples_per_task = 2 - -samples = [ - dict(task_id=task_id, completion=ds.develop_python_functions(sub_problem[task_id]["prompt"])['function_script']) - for task_id in sub_problem - for _ in range(num_samples_per_task) -] -write_jsonl(os.path.join("./.data","samples.jsonl"), samples) \ No newline at end of file + + async def generate_one_completion(prompt): + response = await data_engineer.handle(Message(content=prompt, role="User")) + script = response[-1].data.function_script + return script + + return { + "task_id": task_id, + "completion": await generate_one_completion(prompt) + } +if __name__ == "__main__": + # test_path = "/home/alalulu/workspace/schema-agents/tests/data" + + # import asyncio + # asyncio.run(main()) + problems = read_problems() + def check_correctness(): + # Importing multiprocessing only when this function is called + import multiprocessing + manager = multiprocessing.Manager() + test_path = "/home/alalulu/workspace/schema-agents/tests/data" + # human_path = "/home/alalulu/workspace/human-eval/data" + print(evaluate_functional_correctness(os.path.join(test_path,"samples.jsonl"), [1], problem_file=os.path.join(test_path, "selected_problems.jsonl.gz"))) + check_correctness() \ No newline at end of file From a67a22ff1f9736d02e4a46fc30865efed5bac574 Mon Sep 17 00:00:00 2001 From: alalulu8668 Date: Mon, 13 Nov 2023 04:43:20 +0100 Subject: [PATCH 5/5] clean up --- tests/test_HumanEval.py | 127 ---------------------------------------- 1 file changed, 127 deletions(-) delete mode 100644 tests/test_HumanEval.py diff --git a/tests/test_HumanEval.py b/tests/test_HumanEval.py deleted file mode 100644 index 6bf6839..0000000 --- a/tests/test_HumanEval.py +++ /dev/null @@ -1,127 +0,0 @@ - -import random -import os -import json -from typing import List, Union, Optional, Dict, Any - -from pydantic import BaseModel, Field -from human_eval.evaluation import evaluate_functional_correctness -from human_eval.data import write_jsonl, read_problems - -from schema_agents.tools.code_interpreter import create_mock_client -from schema_agents.schema import Message -from schema_agents.teams.image_analysis_hub.schemas import PythonFunctionScript -from schema_agents.role import Role - -class PythonOutput(BaseModel): - """Represents a Python function with all its properties.""" - function_names: List[str] = Field(..., description="Function names in the script.") - function_script: str = Field(..., description="Completed python function script. This script should be able to pass the test cases. Includes imports, function definition, logic, and implementation.") - docstring: Optional[str] = Field(None, description="Brief notes for usage, debugging, potential error fixing, and further improvements.") - - - - -# async def test_run_python_function( -# role, client, service_id, python_function: PythonFunctionScript -# ) -> PythonFunctionScript: -# """Test run the python function script.""" -# if python_function.pip_packages: -# packages = ",".join([f"'{p}'" for p in python_function.pip_packages]) -# results = await client.executeScript({"script": INSTALL_SCRIPT.format(packages=packages)}) -# output_summary = json.dumps( -# {k: results[k] for k in results.keys() if results[k]}, indent=1 -# ) -# if results["status"] != "ok": -# raise RuntimeError(f"Failed to install pip packages: {python_function.pip_packages}, error: {output_summary}") -# results = await client.executeScript( -# {"script": python_function.function_script + "\n" + python_function.test_script} -# ) - # if results["status"] != "ok": - # output_summary = json.dumps( - # {k: results[k] for k in results.keys() if results[k]}, indent=1 - # ) - # python_function = await fix_code(role, client, python_function, output_summary) - # return await test_run_python_function(role, client, service_id, python_function) - # return python_function - - - -async def generate_code( - req, role: Role -) -> PythonOutput: - """Complete python functions based on the given input.""" - # retrieve req-related memories from long term memory - response = await role.aask(req, PythonOutput) - return response - -async def develop_python_functions( - req: str, role: Role -) -> PythonOutput: - """Complete python functions based on CodeEvalInput to pass the tests.""" - - # if isinstance(req, SoftwareRequirement): - func = await generate_code(req, role) - # try: - # func = await test_run_python_function(role, client, req.id, func) - # except RuntimeError as exp: - # req.additional_notes += f"\nPlease avoid the following error: {exp}" - # func = await generate_code(req, role) - # func = await test_run_python_function(role, client, req.id, func) - - return func - - - -async def main(): - # Your existing code... - - problems = read_problems() - test_path = "/home/alalulu/workspace/schema-agents/tests/data" - - selected_problems = dict(random.sample(problems.items(), 20)) - write_jsonl(os.path.join(test_path, "selected_problems.jsonl.gz"), selected_problems.values()) - - sub_problem = read_problems(os.path.join(test_path, "selected_problems.jsonl.gz")) - - num_samples_per_task = 1 - samples = await asyncio.gather(*[ - generate_sample(task_id, sub_problem[task_id]["prompt"], num_samples_per_task) - for task_id in sub_problem - ]) - - write_jsonl(os.path.join(test_path, "samples.jsonl"), samples) - await evaluate_functional_correctness("/home/alalulu/workspace/schema-agents/tests/data/samples.jsonl", [1], problem_file=os.path.join(test_path, "selected_problems.jsonl.gz")) - -async def generate_sample(task_id, prompt, num_samples): - data_engineer = Role( - name="Alice", - profile="Data Engineer", - goal="Complete the python code script according to the code evaluation input, ensuring that it fulfills the desired functionality. Implement necessary algorithms, handle data processing, and write tests to validate the correctness of the function.", - constraints=None, - actions=[develop_python_functions], - ) - - async def generate_one_completion(prompt): - response = await data_engineer.handle(Message(content=prompt, role="User")) - script = response[-1].data.function_script - return script - - return { - "task_id": task_id, - "completion": await generate_one_completion(prompt) - } -if __name__ == "__main__": - # test_path = "/home/alalulu/workspace/schema-agents/tests/data" - - # import asyncio - # asyncio.run(main()) - problems = read_problems() - def check_correctness(): - # Importing multiprocessing only when this function is called - import multiprocessing - manager = multiprocessing.Manager() - test_path = "/home/alalulu/workspace/schema-agents/tests/data" - # human_path = "/home/alalulu/workspace/human-eval/data" - print(evaluate_functional_correctness(os.path.join(test_path,"samples.jsonl"), [1], problem_file=os.path.join(test_path, "selected_problems.jsonl.gz"))) - check_correctness() \ No newline at end of file