From 5bb6e7cddc5eb8b057e1cc38658fe9a463ad8572 Mon Sep 17 00:00:00 2001 From: Devan Shanker Date: Thu, 25 Jan 2024 00:29:26 -0800 Subject: [PATCH 01/17] Correcting compilation metric --- src/scenicNL/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scenicNL/main.py b/src/scenicNL/main.py index ebabd3c..b499ea6 100644 --- a/src/scenicNL/main.py +++ b/src/scenicNL/main.py @@ -280,8 +280,8 @@ def main( with open(fname, 'w') as f: f.write(output) try: - scenic.scenarioFromFile(fname, mode2D=True) - fname_compile = os.path.join(result_path, f'{fstub}.scenic') + ast = scenic.syntax.parser.parse_file(fname) + print(f'Compiled AST object: {ast}') with open(fname_compile, 'w') as f: f.write(output) print(f'No errors when compiling input {debug}') From a09335b0a94d93d36ab3f8544b4fd88baa8a8c75 Mon Sep 17 00:00:00 2001 From: Devan Shanker Date: Thu, 25 Jan 2024 02:35:39 -0800 Subject: [PATCH 02/17] WIP - need to add support for filling in all {} in few_shot_ast --- src/scenicNL/adapters/anthropic_adapter.py | 65 +++++++++++++++++++++- src/scenicNL/common.py | 20 ++++++- src/scenicNL/main.py | 2 + 3 files changed, 83 insertions(+), 4 deletions(-) diff --git a/src/scenicNL/adapters/anthropic_adapter.py b/src/scenicNL/adapters/anthropic_adapter.py index dce8947..d023f59 100644 --- a/src/scenicNL/adapters/anthropic_adapter.py +++ b/src/scenicNL/adapters/anthropic_adapter.py @@ -6,8 +6,8 @@ import httpx from scenicNL.adapters.model_adapter import ModelAdapter from scenicNL.common import DISCUSSION_TEMPERATURE, NUM_EXPERTS, LLMPromptType, ModelInput, VectorDB, few_shot_prompt_with_rag, get_expert_synthesis_prompt -from scenicNL.common import get_discussion_prompt, get_discussion_to_program_prompt, format_scenic_tutorial_prompt - +from scenicNL.common import get_discussion_prompt, get_discussion_to_program_prompt, format_scenic_tutorial_prompt, get_few_shot_ast_prompt +import scenic class AnthropicModel(Enum): CLAUDE_INSTANT = "claude-instant-1.2" @@ -51,6 +51,7 @@ def _few_shot_prompt( f"{format_scenic_tutorial_prompt()}\n\n" f"Here is the natural language description from the user: \n" f"\n\n{model_input.nat_lang_scene_des}\n\n" + f"\n\nWrite a scenic program that models the natural language description. Provide NO additional commentary before or after. Only output the code." f"\n\n{AI_PROMPT}" ) @@ -176,6 +177,21 @@ def _few_shot_prompt_with_hyde( return prompt + def _few_shot_ast_prompt_( + self, + model_input: ModelInput, + verbose: bool, + top_k: int = 3, + ) -> str: + prompt = get_few_shot_ast_prompt() + + if verbose: + print(f"Anthropic Model {self._model.value}\n" + f"_few_shot_ast_prompt: {prompt}") + + prompt = cast(str, prompt) + return prompt + def _zero_shot_prompt( self, model_input: ModelInput, @@ -281,6 +297,8 @@ def _format_message( msg = self._few_shot_reasoning_hyde(model_input=model_input, verbose=verbose) elif prompt_type == LLMPromptType.EXPERT_SYNTHESIS: msg = self._format_expert_synthesis_prompt(model_input=model_input, verbose=verbose) + elif prompt_type == LLMPromptType.PREDICT_FEW_SHOT_AST: + msg = self._few_shot_ast_prompt_(model_input=model_input, verbose=verbose) else: raise ValueError(f"Invalid prompt type: {prompt_type}") @@ -384,6 +402,49 @@ def _predict( max_tokens_to_sample=max_length_tokens, model=self._model.value, ) + elif prompt_type == LLMPromptType.PREDICT_FEW_SHOT_AST: + # Start with a standard few shot + claude_response = claude.completions.create( + prompt=self._format_message(model_input=model_input, prompt_type=LLMPromptType.PREDICT_FEW_SHOT, verbose=verbose), + temperature=temperature, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + # Up to {retries} retries - depending on compiler feedback + retries = 1 + while retries: + print(f'Retrying... {retries}') + print('\n\n\n^^^^^^^^\n\n\n') + print(claude_response.completion) + with open('_temp.txt', 'w') as f: + f.write(claude_response.completion) + print('\n\n\n^^^^^^^^\n\n\n') + try: + scenic.syntax.parser.parse_file('_temp.txt') + print('No error!') + retries = 0 # If this statement is reached program worked -> terminates loop + except Exception as e: + error_message = str(e) + print(f'Error: {e}') + + # Constructing correcting claude call + new_model_input = ModelInput( + examples=model_input.examples, # this will get overwritten by the search query + nat_lang_scene_des=model_input.nat_lang_scene_des, + first_attempt_scenic_program=str(claude_response.completion), + compiler_error=error_message + ) + # Call claude with few_shot_ast function call type + print(f'\n\n\n%%%%%%%%') + print(self._format_message(model_input=new_model_input, prompt_type=prompt_type, verbose=verbose)) + print(f'%%%%%%%%\n\n\n') + claude_response = claude.completions.create( + prompt=self._format_message(model_input=new_model_input, prompt_type=prompt_type, verbose=verbose), + temperature=temperature, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + retries -= 1 else: claude_response = claude.completions.create( prompt=self._format_message(model_input=model_input, prompt_type=prompt_type, verbose=verbose), diff --git a/src/scenicNL/common.py b/src/scenicNL/common.py index 5dbd28e..4e2b942 100644 --- a/src/scenicNL/common.py +++ b/src/scenicNL/common.py @@ -27,7 +27,7 @@ class LLMPromptType(Enum): PREDICT_TOT_THEN_HYDE = "predict_tot_then_hyde" EXPERT_DISCUSSION = "expert_discussion" EXPERT_SYNTHESIS = "expert_synthesis" - + PREDICT_FEW_SHOT_AST = "predict_few_shot_ast" class PromptFiles(Enum): PROMPT_PATH = os.path.join(os.curdir, 'src', 'scenicNL', 'adapters', 'prompts') @@ -38,7 +38,7 @@ class PromptFiles(Enum): SCENIC_TUTORIAL = os.path.join(PROMPT_PATH, 'scenic_tutorial_prompt.txt') TOT_EXPERT_DISCUSSION = os.path.join(PROMPT_PATH, 'tot_questions.txt') EXPERT_SYNTHESIS = os.path.join(PROMPT_PATH, 'expert_synthesis.txt') - + FEW_SHOT_AST = os.path.join(PROMPT_PATH, 'few_shot_ast.txt') @dataclass(frozen=True) class ModelInput: @@ -50,6 +50,7 @@ class ModelInput: examples: list[str] nat_lang_scene_des: str first_attempt_scenic_program: Optional[str] = None + compiler_error: Optional[str] = None expert_discussion: Optional[str] = None panel_discussion: Optional[List[str]] = None @@ -160,6 +161,21 @@ def get_discussion_to_program_prompt() -> str: return prompt +def get_few_shot_ast_prompt() -> str: + prompt = "" + with open(PromptFiles.FEW_SHOT_AST.value) as f: + prompt = f.read() + + # prompt = prompt.format( + # natural_language_description=model_input.nat_lang_scene_des, + # example_1=model_input.examples[0], + # example_2=model_input.examples[1], + # example_3=model_input.examples[2], + # expert_discussion=model_input.expert_discussion, + # ) + + return prompt + class VectorDB(): def __init__( diff --git a/src/scenicNL/main.py b/src/scenicNL/main.py index b499ea6..73fd4bf 100644 --- a/src/scenicNL/main.py +++ b/src/scenicNL/main.py @@ -280,8 +280,10 @@ def main( with open(fname, 'w') as f: f.write(output) try: + # scenic.scenarioFromFile(fname, mode2D=True) ast = scenic.syntax.parser.parse_file(fname) print(f'Compiled AST object: {ast}') + fname_compile = os.path.join(result_path, f'{fstub}.scenic') with open(fname_compile, 'w') as f: f.write(output) print(f'No errors when compiling input {debug}') From e04f76b4fd3e08d5905ae943528d1831b0c6d71b Mon Sep 17 00:00:00 2001 From: Devan Shanker Date: Thu, 25 Jan 2024 13:40:27 -0800 Subject: [PATCH 03/17] Adding few shot + ast feedback (no rag) - 18% rag 19% rewrite accuracy --- src/scenicNL/adapters/anthropic_adapter.py | 7 +- .../adapters/prompts/few_shot_ast.txt | 120 ++++++++++++++++++ src/scenicNL/common.py | 18 +-- 3 files changed, 134 insertions(+), 11 deletions(-) create mode 100644 src/scenicNL/adapters/prompts/few_shot_ast.txt diff --git a/src/scenicNL/adapters/anthropic_adapter.py b/src/scenicNL/adapters/anthropic_adapter.py index d023f59..b706e0b 100644 --- a/src/scenicNL/adapters/anthropic_adapter.py +++ b/src/scenicNL/adapters/anthropic_adapter.py @@ -183,7 +183,7 @@ def _few_shot_ast_prompt_( verbose: bool, top_k: int = 3, ) -> str: - prompt = get_few_shot_ast_prompt() + prompt = get_few_shot_ast_prompt(model_input=model_input) if verbose: print(f"Anthropic Model {self._model.value}\n" @@ -411,9 +411,8 @@ def _predict( model=self._model.value, ) # Up to {retries} retries - depending on compiler feedback - retries = 1 + retries = 6 while retries: - print(f'Retrying... {retries}') print('\n\n\n^^^^^^^^\n\n\n') print(claude_response.completion) with open('_temp.txt', 'w') as f: @@ -424,6 +423,7 @@ def _predict( print('No error!') retries = 0 # If this statement is reached program worked -> terminates loop except Exception as e: + print(f'Retrying... {retries}') error_message = str(e) print(f'Error: {e}') @@ -445,6 +445,7 @@ def _predict( model=self._model.value, ) retries -= 1 + print(claude_response.completion) else: claude_response = claude.completions.create( prompt=self._format_message(model_input=model_input, prompt_type=prompt_type, verbose=verbose), diff --git a/src/scenicNL/adapters/prompts/few_shot_ast.txt b/src/scenicNL/adapters/prompts/few_shot_ast.txt new file mode 100644 index 0000000..1e08561 --- /dev/null +++ b/src/scenicNL/adapters/prompts/few_shot_ast.txt @@ -0,0 +1,120 @@ + + +Human: +Scenic is a probabilistic programming language for modeling the environments of autonomous cars. A Scenic program defines a distribution over scenes, configurations of physical objects and agents. Scenic can also define (probabilistic) policies for dynamic agents, allowing modeling scenarios where agents take actions over time in response to the state of the world. We use CARLA to render the scenes and simulate the agents. + +Here is one example of a fully compiling Scenic program: +{example_1} + +Create a fully compiling Scenic program that models the description based on: + +1. The following natural language description: +{natural_language_description} + +2. The following scenic program with compiler errors that models the description: +{first_attempt_scenic_program} + +3. The first compiler error raised with the scenic program: +{compiler_error} + +Please output a modified version of scenic_program modified so the compiler error does not appear. + +OUTPUT NO OTHER LEADING OR TRAILING TEXT OR WHITESPACE BESIDES THE CORRECTED SCENIC PROGRAM. NO ONE CARES. + +Scenic references: + +Here is the list of distributions that are supported in Scenic: +Range(low, high) - Uniform distribution over the range [low, high] +DiscreteRange(low, high) - Uniform distribution over the discreet integer range [low, high] +Normal(mean, std) - Normal distribution with mean and standard deviation +TruncatedNormal(mean, stdDev, low, high) - Normal distribution with mean and standard deviation truncated to the range [low, high] +Uniform(value, …) - Uniform distribution over the values provided +Discrete([value: weight, … ]) - Discrete distribution over the values provided with the given weights + +Here are the only behaviors that are allowed for vehicles. Feel free to select more than one as they are composable: +behavior ConstantThrottleBehavior(x : float): +behavior DriveAvoidingCollisions(target_speed : float = 25, avoidance_threshold : float = 10): + # Drive at a target speed, avoiding collisions with other vehicles + # Throttle is off and braking is applied if the distance to the nearest vehicle is less + # than the avoidance threshold +behavior AccelerateForwardBehavior(): # Accelerate forward with throttle set to 0.5 +behavior FollowLaneBehavior(target_speed : float = 10, laneToFollow : Lane = None, is_oppositeTraffic : bool = False): + # Follow's the lane on which the vehicle is at, unless the laneToFollow is specified. + # Once the vehicle reaches an intersection, by default, the vehicle will take the straight route. + # If straight route is not available, then any available turn route will be taken, uniformly randomly. + # If turning at the intersection, the vehicle will slow down to make the turn, safely. + # This behavior does not terminate. A recommended use of the behavior is to accompany it with condition, + # e.g. do FollowLaneBehavior() until ... + # :param target_speed: Its unit is in m/s. By default, it is set to 10 m/s + # :param laneToFollow: If the lane to follow is different from the lane that the vehicle is on, this parameter can be used to specify that lane. By default, this variable will be set to None, which means that the vehicle will follow the lane that it is currently on. +behavior FollowTrajectoryBehavior(target_speed : float = 10, trajectory : List[Lane] = None, turn_speed : float = None): + # Follows the given trajectory. The behavior terminates once the end of the trajectory is reached. + # :param target_speed: Its unit is in m/s. By default, it is set to 10 m/s + # :param trajectory: It is a list of sequential lanes to track, from the lane that the vehicle is initially on to the lane it should end up on. +behavior TurnBehavior(trajectory : List[Lane] = None, target_speed : float = 6): + # This behavior uses a controller specifically tuned for turning at an intersection. + # This behavior is only operational within an intersection, it will terminate if the vehicle is outside of an intersection. +behavior LaneChangeBehavior(laneSectionToSwitchTo : Lane, is_oppositeTraffic : bool = False, target_speed : float = 10): + # is_oppositeTraffic should be specified as True only if the laneSectionToSwitch to has + # the opposite traffic direction to the initial lane from which the vehicle started LaneChangeBehavior + +Here are the only behaviors that are allowed for pedestrians. Feel free to select more than one as they are composable: +behavior WalkForwardBehavior(speed=0.5): + take SetWalkingDirectionAction(self.heading), SetWalkingSpeedAction(speed) + # Walk forward behavior for pedestrians by uniformly sampling either side of the sidewalk for the pedestrian to walk on +behavior WalkBehavior(maxSpeed=1.4): + take SetWalkAction(True, maxSpeed) +behavior CrossingBehavior(reference_actor, min_speed=1, threshold=10, final_speed=None): + # This behavior dynamically controls the speed of an actor that will perpendicularly (or close to) + # cross the road, so that it arrives at a spot in the road at the same time as a reference actor. + # Args: + # min_speed (float): minimum speed of the crossing actor. As this is a type of "synchronization action", + # a minimum speed is needed, to allow the actor to keep moving even if the reference actor has stopped + # threshold (float): starting distance at which the crossing actor starts moving + # final_speed (float): speed of the crossing actor after the reference one surpasses it + +Here is the full set of vehicles supported in Scenic. +"Audi - A2": "vehicle.audi.a2", +"Audi - E-Tron": "vehicle.audi.etron", +"Audi - TT": "vehicle.audi.tt", +"BMW - Gran Tourer": "vehicle.bmw.grandtourer", +"Chevrolet - Impala": "vehicle.chevrolet.impala", +"Citroen - C3": "vehicle.citroen.c3", +"Dodge - Charger 2020": "vehicle.dodge.charger_2020", +"Dodge - Police Charger": "vehicle.dodge.charger_police", +"Dodge - Police Charger 2020": "vehicle.dodge.charger_police_2020", +"Ford - Crown (taxi)": "vehicle.ford.crown", +"Ford - Mustang": "vehicle.ford.mustang", +"Jeep - Wrangler Rubicon": "vehicle.jeep.wrangler_rubicon", +"Lincoln - MKZ 2017": "vehicle.lincoln.mkz_2017", +"Lincoln - MKZ 2020": "vehicle.lincoln.mkz_2020", +"Mercedes - Coupe": "vehicle.mercedes.coupe", +"Mercedes - Coupe 2020": "vehicle.mercedes.coupe_2020", +"Micro - Microlino": "vehicle.micro.microlino", +"Mini - Cooper S": "vehicle.mini.cooper_s", +"Mini - Cooper S 2021": "vehicle.mini.cooper_s_2021", +"Nissan - Micra": "vehicle.nissan.micra", +"Nissan - Patrol": "vehicle.nissan.patrol", +"Nissan - Patrol 2021": "vehicle.nissan.patrol_2021", +"Seat - Leon": "vehicle.seat.leon", +"Tesla - Model 3": "vehicle.tesla.model3", +"Toyota - Prius": "vehicle.toyota.prius", +"CARLA Motors - CarlaCola": "vehicle.carlamotors.carlacola", +"CARLA Motors - European HGV (cab-over-engine type)": "vehicle.carlamotors.european_hgv", +"CARLA Motors - Firetruck": "vehicle.carlamotors.firetruck", +"Tesla - Cybertruck": "vehicle.tesla.cybertruck", +"Ford - Ambulance": "vehicle.ford.ambulance", +"Mercedes - Sprinter": "vehicle.mercedes.sprinter", +"Volkswagen - T2": "vehicle.volkswagen.t2", +"Volkswagen - T2 2021": "vehicle.volkswagen.t2_2021", +"Mitsubishi - Fusorosa": "vehicle.mitsubishi.fusorosa", +"Harley Davidson - Low Rider": "vehicle.harley-davidson.low_rider", +"Kawasaki - Ninja": "vehicle.kawasaki.ninja", +"Vespa - ZX 125": "vehicle.vespa.zx125", +"Yamaha - YZF": "vehicle.yamaha.yzf", +"BH - Crossbike": "vehicle.bh.crossbike", +"Diamondback - Century": "vehicle.diamondback.century", +"Gazelle - Omafiets": "vehicle.gazelle.omafiets" + + +Assistant: \ No newline at end of file diff --git a/src/scenicNL/common.py b/src/scenicNL/common.py index 4e2b942..fa49bdc 100644 --- a/src/scenicNL/common.py +++ b/src/scenicNL/common.py @@ -161,18 +161,20 @@ def get_discussion_to_program_prompt() -> str: return prompt -def get_few_shot_ast_prompt() -> str: +def get_few_shot_ast_prompt(model_input) -> str: prompt = "" with open(PromptFiles.FEW_SHOT_AST.value) as f: prompt = f.read() - # prompt = prompt.format( - # natural_language_description=model_input.nat_lang_scene_des, - # example_1=model_input.examples[0], - # example_2=model_input.examples[1], - # example_3=model_input.examples[2], - # expert_discussion=model_input.expert_discussion, - # ) + prompt = prompt.format( + natural_language_description=model_input.nat_lang_scene_des, + example_1=model_input.examples[0], + example_2=model_input.examples[1], + example_3=model_input.examples[2], + expert_discussion=model_input.expert_discussion, + first_attempt_scenic_program=model_input.first_attempt_scenic_program, + compiler_error=model_input.compiler_error + ) return prompt From 64a98ea59c0dc52e041638d0b43398c358508c1b Mon Sep 17 00:00:00 2001 From: Devan Shanker Date: Thu, 25 Jan 2024 15:28:00 -0800 Subject: [PATCH 04/17] Updates to prompting and error details -> 56% on 100 examples for 6 retries --- src/scenicNL/adapters/anthropic_adapter.py | 15 +++++++++------ src/scenicNL/adapters/prompts/few_shot_ast.txt | 1 + 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/scenicNL/adapters/anthropic_adapter.py b/src/scenicNL/adapters/anthropic_adapter.py index b706e0b..c86f1a2 100644 --- a/src/scenicNL/adapters/anthropic_adapter.py +++ b/src/scenicNL/adapters/anthropic_adapter.py @@ -413,11 +413,11 @@ def _predict( # Up to {retries} retries - depending on compiler feedback retries = 6 while retries: - print('\n\n\n^^^^^^^^\n\n\n') - print(claude_response.completion) + # print('\n\n\n^^^^^^^^\n\n\n') + # print(claude_response.completion) + # print('\n\n\n^^^^^^^^\n\n\n') with open('_temp.txt', 'w') as f: f.write(claude_response.completion) - print('\n\n\n^^^^^^^^\n\n\n') try: scenic.syntax.parser.parse_file('_temp.txt') print('No error!') @@ -425,7 +425,10 @@ def _predict( except Exception as e: print(f'Retrying... {retries}') error_message = str(e) + print(error_message) + error_message = f"""Error details below..\nmessage: {str(e)}\ntext: {e.text}\nlineno: {e.lineno}\nend_lineno: {e.end_lineno}\noffset: {e.offset}\nend_offset: {e.end_offset}""" print(f'Error: {e}') + print(error_message) # Constructing correcting claude call new_model_input = ModelInput( @@ -435,9 +438,9 @@ def _predict( compiler_error=error_message ) # Call claude with few_shot_ast function call type - print(f'\n\n\n%%%%%%%%') - print(self._format_message(model_input=new_model_input, prompt_type=prompt_type, verbose=verbose)) - print(f'%%%%%%%%\n\n\n') + # print(f'\n\n\n%%%%%%%%') + # print(self._format_message(model_input=new_model_input, prompt_type=prompt_type, verbose=verbose)) + # print(f'%%%%%%%%\n\n\n') claude_response = claude.completions.create( prompt=self._format_message(model_input=new_model_input, prompt_type=prompt_type, verbose=verbose), temperature=temperature, diff --git a/src/scenicNL/adapters/prompts/few_shot_ast.txt b/src/scenicNL/adapters/prompts/few_shot_ast.txt index 1e08561..cefd117 100644 --- a/src/scenicNL/adapters/prompts/few_shot_ast.txt +++ b/src/scenicNL/adapters/prompts/few_shot_ast.txt @@ -20,6 +20,7 @@ Create a fully compiling Scenic program that models the description based on: Please output a modified version of scenic_program modified so the compiler error does not appear. OUTPUT NO OTHER LEADING OR TRAILING TEXT OR WHITESPACE BESIDES THE CORRECTED SCENIC PROGRAM. NO ONE CARES. +Output only the natural language description of the scenario as a comment at the top of the file and your proposed Scenic program. DO NOT output any other text or whitespace so I can run your program and see the results. Scenic references: From e74fde513f81ef4c1b411c1fb6a592a38cdb439e Mon Sep 17 00:00:00 2001 From: Devan Shanker Date: Thu, 25 Jan 2024 20:02:10 -0800 Subject: [PATCH 05/17] Support for general retries --- src/scenicNL/adapters/anthropic_adapter.py | 115 ++++++++++++++------- 1 file changed, 75 insertions(+), 40 deletions(-) diff --git a/src/scenicNL/adapters/anthropic_adapter.py b/src/scenicNL/adapters/anthropic_adapter.py index c86f1a2..8b2ff8b 100644 --- a/src/scenicNL/adapters/anthropic_adapter.py +++ b/src/scenicNL/adapters/anthropic_adapter.py @@ -4,6 +4,7 @@ from anthropic import Anthropic, AI_PROMPT, HUMAN_PROMPT import httpx +import os from scenicNL.adapters.model_adapter import ModelAdapter from scenicNL.common import DISCUSSION_TEMPERATURE, NUM_EXPERTS, LLMPromptType, ModelInput, VectorDB, few_shot_prompt_with_rag, get_expert_synthesis_prompt from scenicNL.common import get_discussion_prompt, get_discussion_to_program_prompt, format_scenic_tutorial_prompt, get_few_shot_ast_prompt @@ -314,7 +315,7 @@ def _predict( temperature: float, max_length_tokens: int, prompt_type: LLMPromptType, - verbose: bool + verbose: bool, ) -> str: # to prevent misuse of file handlers limits = httpx.Limits(max_keepalive_connections=1, max_connections=1) @@ -411,44 +412,42 @@ def _predict( model=self._model.value, ) # Up to {retries} retries - depending on compiler feedback - retries = 6 - while retries: - # print('\n\n\n^^^^^^^^\n\n\n') - # print(claude_response.completion) - # print('\n\n\n^^^^^^^^\n\n\n') - with open('_temp.txt', 'w') as f: - f.write(claude_response.completion) - try: - scenic.syntax.parser.parse_file('_temp.txt') - print('No error!') - retries = 0 # If this statement is reached program worked -> terminates loop - except Exception as e: - print(f'Retrying... {retries}') - error_message = str(e) - print(error_message) - error_message = f"""Error details below..\nmessage: {str(e)}\ntext: {e.text}\nlineno: {e.lineno}\nend_lineno: {e.end_lineno}\noffset: {e.offset}\nend_offset: {e.end_offset}""" - print(f'Error: {e}') - print(error_message) - - # Constructing correcting claude call - new_model_input = ModelInput( - examples=model_input.examples, # this will get overwritten by the search query - nat_lang_scene_des=model_input.nat_lang_scene_des, - first_attempt_scenic_program=str(claude_response.completion), - compiler_error=error_message - ) - # Call claude with few_shot_ast function call type - # print(f'\n\n\n%%%%%%%%') - # print(self._format_message(model_input=new_model_input, prompt_type=prompt_type, verbose=verbose)) - # print(f'%%%%%%%%\n\n\n') - claude_response = claude.completions.create( - prompt=self._format_message(model_input=new_model_input, prompt_type=prompt_type, verbose=verbose), - temperature=temperature, - max_tokens_to_sample=max_length_tokens, - model=self._model.value, - ) - retries -= 1 - print(claude_response.completion) + if False: + retries = 0 + while retries: + # print('\n\n\n^^^^^^^^\n\n\n') + # print(claude_response.completion) + # print('\n\n\n^^^^^^^^\n\n\n') + with open('_temp.txt', 'w') as f: + f.write(claude_response.completion) + try: + scenic.syntax.parser.parse_file('_temp.txt') + print('No error!') + retries = 0 # If this statement is reached program worked -> terminates loop + except Exception as e: + print(f'Retrying... {retries}') + error_message = f"Error details below..\nmessage: {str(e)}\ntext: {e.text}\nlineno: {e.lineno}\nend_lineno: {e.end_lineno}\noffset: {e.offset}\nend_offset: {e.end_offset}" + print(error_message) + + # Constructing correcting claude call + new_model_input = ModelInput( + examples=model_input.examples, # this will get overwritten by the search query + nat_lang_scene_des=model_input.nat_lang_scene_des, + first_attempt_scenic_program=str(claude_response.completion), + compiler_error=error_message + ) + # Call claude with few_shot_ast function call type + # print(f'\n\n\n%%%%%%%%') + # print(self._format_message(model_input=new_model_input, prompt_type=prompt_type, verbose=verbose)) + # print(f'%%%%%%%%\n\n\n') + claude_response = claude.completions.create( + prompt=self._format_message(model_input=new_model_input, prompt_type=prompt_type, verbose=verbose), + temperature=temperature, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + retries -= 1 + print(claude_response.completion) else: claude_response = claude.completions.create( prompt=self._format_message(model_input=model_input, prompt_type=prompt_type, verbose=verbose), @@ -456,6 +455,42 @@ def _predict( max_tokens_to_sample=max_length_tokens, model=self._model.value, ) - + + retries = 6 + while retries: + # print('\n\n\n^^^^^^^^\n\n\n') + # print(claude_response.completion) + # print('\n\n\n^^^^^^^^\n\n\n') + with open('_temp.txt', 'w') as f: + f.write(claude_response.completion) + try: + scenic.syntax.parser.parse_file('_temp.txt') + print('No error!') + retries = 0 # If this statement is reached program worked -> terminates loop + except Exception as e: + print(f'Retrying... {retries}') + error_message = f"Error details below..\nmessage: {str(e)}\ntext: {e.text}\nlineno: {e.lineno}\nend_lineno: {e.end_lineno}\noffset: {e.offset}\nend_offset: {e.end_offset}" + print(error_message) + + # Constructing correcting claude call + new_model_input = ModelInput( + examples=model_input.examples, # this will get overwritten by the search query + nat_lang_scene_des=model_input.nat_lang_scene_des, + first_attempt_scenic_program=str(claude_response.completion), + compiler_error=error_message + ) + # Call claude with few_shot_ast function call type + # print(f'\n\n\n%%%%%%%%') + # print(self._format_message(model_input=new_model_input, prompt_type=prompt_type, verbose=verbose)) + # print(f'%%%%%%%%\n\n\n') + claude_response = claude.completions.create( + prompt=self._format_message(model_input=new_model_input, prompt_type=prompt_type, verbose=verbose), + temperature=temperature, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + retries -= 1 + # os.remove('_temp.txt') + print(claude_response.completion) return claude_response.completion \ No newline at end of file From 046b0bb2fb050d2db4ecf9d1638ea668b9d2bdc1 Mon Sep 17 00:00:00 2001 From: Devan Shanker Date: Thu, 25 Jan 2024 22:21:04 -0800 Subject: [PATCH 06/17] Adding compile and execute rates + max_retries parameter to main --- src/scenicNL/adapters/anthropic_adapter.py | 67 +++------------------- src/scenicNL/adapters/model_adapter.py | 7 ++- src/scenicNL/main.py | 43 ++++++++++---- 3 files changed, 48 insertions(+), 69 deletions(-) diff --git a/src/scenicNL/adapters/anthropic_adapter.py b/src/scenicNL/adapters/anthropic_adapter.py index 8b2ff8b..5d7bf22 100644 --- a/src/scenicNL/adapters/anthropic_adapter.py +++ b/src/scenicNL/adapters/anthropic_adapter.py @@ -316,7 +316,9 @@ def _predict( max_length_tokens: int, prompt_type: LLMPromptType, verbose: bool, + max_retries: int ) -> str: + verbose_retry = True # to prevent misuse of file handlers limits = httpx.Limits(max_keepalive_connections=1, max_connections=1) with Anthropic(connection_pool_limits=limits, max_retries=10) as claude: @@ -403,51 +405,6 @@ def _predict( max_tokens_to_sample=max_length_tokens, model=self._model.value, ) - elif prompt_type == LLMPromptType.PREDICT_FEW_SHOT_AST: - # Start with a standard few shot - claude_response = claude.completions.create( - prompt=self._format_message(model_input=model_input, prompt_type=LLMPromptType.PREDICT_FEW_SHOT, verbose=verbose), - temperature=temperature, - max_tokens_to_sample=max_length_tokens, - model=self._model.value, - ) - # Up to {retries} retries - depending on compiler feedback - if False: - retries = 0 - while retries: - # print('\n\n\n^^^^^^^^\n\n\n') - # print(claude_response.completion) - # print('\n\n\n^^^^^^^^\n\n\n') - with open('_temp.txt', 'w') as f: - f.write(claude_response.completion) - try: - scenic.syntax.parser.parse_file('_temp.txt') - print('No error!') - retries = 0 # If this statement is reached program worked -> terminates loop - except Exception as e: - print(f'Retrying... {retries}') - error_message = f"Error details below..\nmessage: {str(e)}\ntext: {e.text}\nlineno: {e.lineno}\nend_lineno: {e.end_lineno}\noffset: {e.offset}\nend_offset: {e.end_offset}" - print(error_message) - - # Constructing correcting claude call - new_model_input = ModelInput( - examples=model_input.examples, # this will get overwritten by the search query - nat_lang_scene_des=model_input.nat_lang_scene_des, - first_attempt_scenic_program=str(claude_response.completion), - compiler_error=error_message - ) - # Call claude with few_shot_ast function call type - # print(f'\n\n\n%%%%%%%%') - # print(self._format_message(model_input=new_model_input, prompt_type=prompt_type, verbose=verbose)) - # print(f'%%%%%%%%\n\n\n') - claude_response = claude.completions.create( - prompt=self._format_message(model_input=new_model_input, prompt_type=prompt_type, verbose=verbose), - temperature=temperature, - max_tokens_to_sample=max_length_tokens, - model=self._model.value, - ) - retries -= 1 - print(claude_response.completion) else: claude_response = claude.completions.create( prompt=self._format_message(model_input=model_input, prompt_type=prompt_type, verbose=verbose), @@ -456,21 +413,18 @@ def _predict( model=self._model.value, ) - retries = 6 + retries = max_retries while retries: - # print('\n\n\n^^^^^^^^\n\n\n') - # print(claude_response.completion) - # print('\n\n\n^^^^^^^^\n\n\n') with open('_temp.txt', 'w') as f: f.write(claude_response.completion) try: scenic.syntax.parser.parse_file('_temp.txt') - print('No error!') + if verbose_retry: print('No error!') retries = 0 # If this statement is reached program worked -> terminates loop except Exception as e: - print(f'Retrying... {retries}') + if verbose_retry: print(f'Retrying... {retries}') error_message = f"Error details below..\nmessage: {str(e)}\ntext: {e.text}\nlineno: {e.lineno}\nend_lineno: {e.end_lineno}\noffset: {e.offset}\nend_offset: {e.end_offset}" - print(error_message) + if verbose_retry: print(error_message) # Constructing correcting claude call new_model_input = ModelInput( @@ -479,10 +433,7 @@ def _predict( first_attempt_scenic_program=str(claude_response.completion), compiler_error=error_message ) - # Call claude with few_shot_ast function call type - # print(f'\n\n\n%%%%%%%%') - # print(self._format_message(model_input=new_model_input, prompt_type=prompt_type, verbose=verbose)) - # print(f'%%%%%%%%\n\n\n') + claude_response = claude.completions.create( prompt=self._format_message(model_input=new_model_input, prompt_type=prompt_type, verbose=verbose), temperature=temperature, @@ -490,7 +441,7 @@ def _predict( model=self._model.value, ) retries -= 1 - # os.remove('_temp.txt') - print(claude_response.completion) + if os.path.exists('_temp.txt'): os.remove('_temp.txt') + if verbose_retry: print(claude_response.completion) return claude_response.completion \ No newline at end of file diff --git a/src/scenicNL/adapters/model_adapter.py b/src/scenicNL/adapters/model_adapter.py index 15ad0c1..b9c69cf 100644 --- a/src/scenicNL/adapters/model_adapter.py +++ b/src/scenicNL/adapters/model_adapter.py @@ -49,6 +49,7 @@ def _predict( max_length_tokens: int, prompt_type: LLMPromptType, verbose: bool, + max_retries: int, ) -> str: """Perform a single prediction. Must be overriden by subclasses.""" raise NotImplementedError @@ -80,7 +81,8 @@ def _batch_processor( cache: Cache, prompt_type: LLMPromptType, ignore_cache: bool, - verbose: bool = False, + verbose: bool, + max_retries: int, ) -> Callable[[ModelInput], list[str | APIError]]: """ Return a function that takes a list of model inputs and returns a @@ -114,6 +116,7 @@ def process_single( max_length_tokens=max_tokens, prompt_type=prompt_type, verbose=verbose, + max_retries=max_retries, ) if prompt_type.value == LLMPromptType.PREDICT_PYTHON_API.value: api_input = ModelInput(model_input.examples, prediction) @@ -146,6 +149,7 @@ def predict_batch( verbose: bool = False, num_workers: int = 10, ignore_cache: bool = False, + max_retries: int = 0, ) -> Iterable[list[str | APIError]]: """ Given a stream of model inputs, return a stream of predictions. This @@ -168,6 +172,7 @@ def predict_batch( prompt_type=prompt_type, ignore_cache=ignore_cache, verbose=verbose, + max_retries=max_retries, ) with ThreadPool(num_workers) as pool: diff --git a/src/scenicNL/main.py b/src/scenicNL/main.py index 73fd4bf..ab555ad 100644 --- a/src/scenicNL/main.py +++ b/src/scenicNL/main.py @@ -161,6 +161,14 @@ def main(): help="Number of workers to use for parallel processing.", ) +@click.option( + "--max_retries", + type=click.INT, + default=0, + show_default=True, + help="Maximum number of compiler-in-the-loop retries.", +) + def main( query_path: Path, output_path: Path, @@ -178,6 +186,7 @@ def main( should_cache_retry_errors: bool, keep_filename: bool, temperature: float, + max_retries: int, ) -> None: """ Generate simulator scenes from natural language descriptions. @@ -248,7 +257,7 @@ def main( scenic_metric_path = os.path.join(scenic_metadata_path, 'metrics.txt') start_time = time.time() - compile_pass, compile_fail, api_error = 0, 0, 0 + compile_pass = compile_fail = execute_pass = execute_fail = api_error = 0 for index, outputs in enumerate(adapter.predict_batch( model_inputs=model_input_list, cache_path=cache_path, @@ -260,6 +269,7 @@ def main( verbose=verbose, num_workers=num_workers, ignore_cache=ignore_cache, + max_retries=max_retries, ) ): for attempt, output in enumerate(outputs): @@ -280,19 +290,29 @@ def main( with open(fname, 'w') as f: f.write(output) try: - # scenic.scenarioFromFile(fname, mode2D=True) ast = scenic.syntax.parser.parse_file(fname) - print(f'Compiled AST object: {ast}') - fname_compile = os.path.join(result_path, f'{fstub}.scenic') - with open(fname_compile, 'w') as f: - f.write(output) - print(f'No errors when compiling input {debug}') + print(f'Compiled input {debug} successfully: {ast}') + # print(f'No errors when compiling input {debug}') compile_pass += 1 except Exception as e: print(f'Error while compiling for input {debug}: {e}') compile_fail += 1 with open(scenic_error_path, 'a') as f: - f.write(f'{index} - {fstub}: {e}\n') + f.write(f'{index} - {fstub} compile error: {e}\n') + try: + scenario = scenic.scenarioFromFile(fname, mode2D=True) + print(f'Executed input {debug} successfully: {scenario}') + + fsave = os.path.join(result_path, f'{fstub}.scenic') + execute_pass += 1 + with open(fsave, 'w') as f: + f.write(output) + # print(f'No errors when compiling input {debug}') + except Exception as e: + print(f'Error while compiling for input {debug}: {e}') + execute_fail += 1 + with open(scenic_error_path, 'a') as f: + f.write(f'{index} - {fstub} execute error: {e}\n') print('----------------\n\n') end_time = time.time() @@ -300,13 +320,16 @@ def main( api_error_rate = round((100*api_error/total), 2) compile_rate = round((100*compile_pass/total), 2) + execute_rate = round((100*execute_pass/total), 2) eval_rate = round((end_time-start_time)/total, 5) - print(f'API error rate: {api_error_rate}%') - print(f'Compilation success rate: {compile_rate}%') + print(f'Compile success rate: {compile_rate}%') + print(f'Execute success rate: {execute_rate}%') + print(f'## API error rate ##: {api_error_rate}%') print(scenic_metric_path) with open(scenic_metric_path, 'w') as f: f.write(f'Compile rate: {compile_rate}\n') + f.write(f'Execute rate: {execute_rate}\n') f.write(f'API error rate: {api_error_rate}\n') f.write(f'Secs per program: {eval_rate}\n') From 289f509ff5dce3f820328ed12ebacc3f1853f2e6 Mon Sep 17 00:00:00 2001 From: Devan Shanker Date: Fri, 26 Jan 2024 11:47:19 -0800 Subject: [PATCH 07/17] Adding compilation and execution feedback (compilation first), adding support for valid filepaths with nested directory --- src/scenicNL/adapters/anthropic_adapter.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/scenicNL/adapters/anthropic_adapter.py b/src/scenicNL/adapters/anthropic_adapter.py index 5d7bf22..d0291ff 100644 --- a/src/scenicNL/adapters/anthropic_adapter.py +++ b/src/scenicNL/adapters/anthropic_adapter.py @@ -5,6 +5,7 @@ from anthropic import Anthropic, AI_PROMPT, HUMAN_PROMPT import httpx import os +from pathlib import Path from scenicNL.adapters.model_adapter import ModelAdapter from scenicNL.common import DISCUSSION_TEMPERATURE, NUM_EXPERTS, LLMPromptType, ModelInput, VectorDB, few_shot_prompt_with_rag, get_expert_synthesis_prompt from scenicNL.common import get_discussion_prompt, get_discussion_to_program_prompt, format_scenic_tutorial_prompt, get_few_shot_ast_prompt @@ -413,18 +414,24 @@ def _predict( model=self._model.value, ) - retries = max_retries + retries, retries_dir, fstub = max_retries, os.path.join(os.curdir, 'retries', 'retries'), 'temp.scenic' + Path(retries_dir).mkdir(parents=True, exist_ok=True) + fname = os.path.join(retries_dir, fstub) while retries: - with open('_temp.txt', 'w') as f: + with open(fname, 'w') as f: f.write(claude_response.completion) try: - scenic.syntax.parser.parse_file('_temp.txt') + ast = scenic.syntax.parser.parse_file(fname) + scenario = scenic.scenarioFromFile(fname, mode2D=True) if verbose_retry: print('No error!') retries = 0 # If this statement is reached program worked -> terminates loop except Exception as e: if verbose_retry: print(f'Retrying... {retries}') - error_message = f"Error details below..\nmessage: {str(e)}\ntext: {e.text}\nlineno: {e.lineno}\nend_lineno: {e.end_lineno}\noffset: {e.offset}\nend_offset: {e.end_offset}" - if verbose_retry: print(error_message) + try: + error_message = f"Error details below..\nmessage: {str(e)}\ntext: {e.text}\nlineno: {e.lineno}\nend_lineno: {e.end_lineno}\noffset: {e.offset}\nend_offset: {e.end_offset}" + if verbose_retry: print(error_message) + except: + error_message = f'Error details below..\n' # Constructing correcting claude call new_model_input = ModelInput( @@ -441,7 +448,7 @@ def _predict( model=self._model.value, ) retries -= 1 - if os.path.exists('_temp.txt'): os.remove('_temp.txt') + if os.path.exists(fname): os.remove(fname) if verbose_retry: print(claude_response.completion) return claude_response.completion \ No newline at end of file From 5ac69a003c332e876d343baf5e6a49e1594a0829 Mon Sep 17 00:00:00 2001 From: Devan Shanker Date: Fri, 26 Jan 2024 12:04:12 -0800 Subject: [PATCH 08/17] Correcting error messages and string formatting --- src/scenicNL/adapters/anthropic_adapter.py | 3 ++- src/scenicNL/main.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/scenicNL/adapters/anthropic_adapter.py b/src/scenicNL/adapters/anthropic_adapter.py index d0291ff..98dab38 100644 --- a/src/scenicNL/adapters/anthropic_adapter.py +++ b/src/scenicNL/adapters/anthropic_adapter.py @@ -431,7 +431,8 @@ def _predict( error_message = f"Error details below..\nmessage: {str(e)}\ntext: {e.text}\nlineno: {e.lineno}\nend_lineno: {e.end_lineno}\noffset: {e.offset}\nend_offset: {e.end_offset}" if verbose_retry: print(error_message) except: - error_message = f'Error details below..\n' + error_message = f'Error details below..\nmessage: {str(e)}' + if verbose_retry: print(error_message) # Constructing correcting claude call new_model_input = ModelInput( diff --git a/src/scenicNL/main.py b/src/scenicNL/main.py index ab555ad..5970c4f 100644 --- a/src/scenicNL/main.py +++ b/src/scenicNL/main.py @@ -309,7 +309,7 @@ def main( f.write(output) # print(f'No errors when compiling input {debug}') except Exception as e: - print(f'Error while compiling for input {debug}: {e}') + print(f'Error while executing for input {debug}: {e}') execute_fail += 1 with open(scenic_error_path, 'a') as f: f.write(f'{index} - {fstub} execute error: {e}\n') From dfdef97ad65bf9b8e138211e8b0474d01fe53e29 Mon Sep 17 00:00:00 2001 From: Devan Shanker Date: Sat, 27 Jan 2024 15:11:33 -0800 Subject: [PATCH 09/17] Renaming ast feedback for claude --- src/scenicNL/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/scenicNL/common.py b/src/scenicNL/common.py index fa49bdc..158c11b 100644 --- a/src/scenicNL/common.py +++ b/src/scenicNL/common.py @@ -27,7 +27,7 @@ class LLMPromptType(Enum): PREDICT_TOT_THEN_HYDE = "predict_tot_then_hyde" EXPERT_DISCUSSION = "expert_discussion" EXPERT_SYNTHESIS = "expert_synthesis" - PREDICT_FEW_SHOT_AST = "predict_few_shot_ast" + AST_FEEDBACK = "ast_feedback" class PromptFiles(Enum): PROMPT_PATH = os.path.join(os.curdir, 'src', 'scenicNL', 'adapters', 'prompts') @@ -38,7 +38,7 @@ class PromptFiles(Enum): SCENIC_TUTORIAL = os.path.join(PROMPT_PATH, 'scenic_tutorial_prompt.txt') TOT_EXPERT_DISCUSSION = os.path.join(PROMPT_PATH, 'tot_questions.txt') EXPERT_SYNTHESIS = os.path.join(PROMPT_PATH, 'expert_synthesis.txt') - FEW_SHOT_AST = os.path.join(PROMPT_PATH, 'few_shot_ast.txt') + AST_FEEDBACK_CLAUDE = os.path.join(PROMPT_PATH, 'few_shot_ast.txt') @dataclass(frozen=True) class ModelInput: @@ -163,7 +163,7 @@ def get_discussion_to_program_prompt() -> str: def get_few_shot_ast_prompt(model_input) -> str: prompt = "" - with open(PromptFiles.FEW_SHOT_AST.value) as f: + with open(PromptFiles.AST_FEEDBACK_CLAUDE.value) as f: prompt = f.read() prompt = prompt.format( From 1d335f27dab23091379bfb1399d58b94e656dd42 Mon Sep 17 00:00:00 2001 From: Devan Shanker Date: Sat, 27 Jan 2024 15:11:48 -0800 Subject: [PATCH 10/17] Renaming pt 2 --- src/scenicNL/adapters/anthropic_adapter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/scenicNL/adapters/anthropic_adapter.py b/src/scenicNL/adapters/anthropic_adapter.py index 98dab38..7e33e5e 100644 --- a/src/scenicNL/adapters/anthropic_adapter.py +++ b/src/scenicNL/adapters/anthropic_adapter.py @@ -179,7 +179,7 @@ def _few_shot_prompt_with_hyde( return prompt - def _few_shot_ast_prompt_( + def _ast_feedback_prompt( self, model_input: ModelInput, verbose: bool, @@ -299,8 +299,8 @@ def _format_message( msg = self._few_shot_reasoning_hyde(model_input=model_input, verbose=verbose) elif prompt_type == LLMPromptType.EXPERT_SYNTHESIS: msg = self._format_expert_synthesis_prompt(model_input=model_input, verbose=verbose) - elif prompt_type == LLMPromptType.PREDICT_FEW_SHOT_AST: - msg = self._few_shot_ast_prompt_(model_input=model_input, verbose=verbose) + elif prompt_type == LLMPromptType.AST_FEEDBACK: + msg = self._ast_feedback_prompt(model_input=model_input, verbose=verbose) else: raise ValueError(f"Invalid prompt type: {prompt_type}") From c796aa1f85d7715ab73c5ccd50fc733323fd0141 Mon Sep 17 00:00:00 2001 From: Devan Shanker Date: Sat, 27 Jan 2024 19:58:27 -0800 Subject: [PATCH 11/17] Temp file and dir wrappers - fix file naming and remove ^^^ --- src/scenicNL/adapters/anthropic_adapter.py | 79 ++++++++++++---------- 1 file changed, 43 insertions(+), 36 deletions(-) diff --git a/src/scenicNL/adapters/anthropic_adapter.py b/src/scenicNL/adapters/anthropic_adapter.py index 7e33e5e..1375fae 100644 --- a/src/scenicNL/adapters/anthropic_adapter.py +++ b/src/scenicNL/adapters/anthropic_adapter.py @@ -10,6 +10,7 @@ from scenicNL.common import DISCUSSION_TEMPERATURE, NUM_EXPERTS, LLMPromptType, ModelInput, VectorDB, few_shot_prompt_with_rag, get_expert_synthesis_prompt from scenicNL.common import get_discussion_prompt, get_discussion_to_program_prompt, format_scenic_tutorial_prompt, get_few_shot_ast_prompt import scenic +import tempfile class AnthropicModel(Enum): CLAUDE_INSTANT = "claude-instant-1.2" @@ -414,42 +415,48 @@ def _predict( model=self._model.value, ) - retries, retries_dir, fstub = max_retries, os.path.join(os.curdir, 'retries', 'retries'), 'temp.scenic' - Path(retries_dir).mkdir(parents=True, exist_ok=True) - fname = os.path.join(retries_dir, fstub) - while retries: - with open(fname, 'w') as f: - f.write(claude_response.completion) - try: - ast = scenic.syntax.parser.parse_file(fname) - scenario = scenic.scenarioFromFile(fname, mode2D=True) - if verbose_retry: print('No error!') - retries = 0 # If this statement is reached program worked -> terminates loop - except Exception as e: - if verbose_retry: print(f'Retrying... {retries}') - try: - error_message = f"Error details below..\nmessage: {str(e)}\ntext: {e.text}\nlineno: {e.lineno}\nend_lineno: {e.end_lineno}\noffset: {e.offset}\nend_offset: {e.end_offset}" - if verbose_retry: print(error_message) - except: - error_message = f'Error details below..\nmessage: {str(e)}' - if verbose_retry: print(error_message) - - # Constructing correcting claude call - new_model_input = ModelInput( - examples=model_input.examples, # this will get overwritten by the search query - nat_lang_scene_des=model_input.nat_lang_scene_des, - first_attempt_scenic_program=str(claude_response.completion), - compiler_error=error_message - ) - - claude_response = claude.completions.create( - prompt=self._format_message(model_input=new_model_input, prompt_type=prompt_type, verbose=verbose), - temperature=temperature, - max_tokens_to_sample=max_length_tokens, - model=self._model.value, - ) - retries -= 1 - if os.path.exists(fname): os.remove(fname) + with tempfile.TemporaryDirectory(dir=os.curdir) as temp_dir: + retries = max_retries + retries_dir = os.path.join(temp_dir, 'temp_dir') + print(f'^^^: {temp_dir}') + os.makedirs(retries_dir) + + with tempfile.NamedTemporaryFile(dir=retries_dir, delete=False, suffix='.scenic') as temp_file: + fname = temp_file.name + print(f'$$$: {fname}') + + while retries: + with open(fname, 'w') as f: + f.write(claude_response.completion) + try: + ast = scenic.syntax.parser.parse_file(fname) + scenario = scenic.scenarioFromFile(fname, mode2D=True) + if verbose_retry: print('No error!') + retries = 0 # If this statement is reached program worked -> terminates loop + except Exception as e: + if verbose_retry: print(f'Retrying... {retries}') + try: + error_message = f"Error details below..\nmessage: {str(e)}\ntext: {e.text}\nlineno: {e.lineno}\nend_lineno: {e.end_lineno}\noffset: {e.offset}\nend_offset: {e.end_offset}" + if verbose_retry: print(error_message) + except: + error_message = f'Error details below..\nmessage: {str(e)}' + if verbose_retry: print(error_message) + + # Constructing correcting claude call + new_model_input = ModelInput( + examples=model_input.examples, # this will get overwritten by the search query + nat_lang_scene_des=model_input.nat_lang_scene_des, + first_attempt_scenic_program=str(claude_response.completion), + compiler_error=error_message + ) + + claude_response = claude.completions.create( + prompt=self._format_message(model_input=new_model_input, prompt_type=prompt_type, verbose=verbose), + temperature=temperature, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + retries -= 1 if verbose_retry: print(claude_response.completion) return claude_response.completion \ No newline at end of file From d9eeb86eb3464d87da95da85802cfd0cd1c2070e Mon Sep 17 00:00:00 2001 From: Devan Shanker Date: Mon, 29 Jan 2024 02:54:44 -0800 Subject: [PATCH 12/17] Addressing whitespace leading on first param map line --- src/scenicNL/adapters/anthropic_adapter.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/scenicNL/adapters/anthropic_adapter.py b/src/scenicNL/adapters/anthropic_adapter.py index 1375fae..38ae974 100644 --- a/src/scenicNL/adapters/anthropic_adapter.py +++ b/src/scenicNL/adapters/anthropic_adapter.py @@ -9,6 +9,7 @@ from scenicNL.adapters.model_adapter import ModelAdapter from scenicNL.common import DISCUSSION_TEMPERATURE, NUM_EXPERTS, LLMPromptType, ModelInput, VectorDB, few_shot_prompt_with_rag, get_expert_synthesis_prompt from scenicNL.common import get_discussion_prompt, get_discussion_to_program_prompt, format_scenic_tutorial_prompt, get_few_shot_ast_prompt +import re import scenic import tempfile @@ -415,6 +416,8 @@ def _predict( model=self._model.value, ) + model_result = str(claude_response.completion) + with tempfile.TemporaryDirectory(dir=os.curdir) as temp_dir: retries = max_retries retries_dir = os.path.join(temp_dir, 'temp_dir') @@ -427,9 +430,12 @@ def _predict( while retries: with open(fname, 'w') as f: - f.write(claude_response.completion) + pattern = r'\s+param map' + replacement = r'\nparam map' + model_result = re.sub(pattern, replacement, model_result) + f.write(model_result) try: - ast = scenic.syntax.parser.parse_file(fname) + # ast = scenic.syntax.parser.parse_file(fname) scenario = scenic.scenarioFromFile(fname, mode2D=True) if verbose_retry: print('No error!') retries = 0 # If this statement is reached program worked -> terminates loop @@ -446,7 +452,7 @@ def _predict( new_model_input = ModelInput( examples=model_input.examples, # this will get overwritten by the search query nat_lang_scene_des=model_input.nat_lang_scene_des, - first_attempt_scenic_program=str(claude_response.completion), + first_attempt_scenic_program=str(model_result), compiler_error=error_message ) @@ -456,7 +462,8 @@ def _predict( max_tokens_to_sample=max_length_tokens, model=self._model.value, ) + model_result = str(claude_response.completion) retries -= 1 - if verbose_retry: print(claude_response.completion) - return claude_response.completion + if verbose_retry: print(model_result) + return model_result \ No newline at end of file From 628b23a4a21339b4bf9d73be5355624bda1bc334 Mon Sep 17 00:00:00 2001 From: Devan Shanker Date: Mon, 29 Jan 2024 03:53:20 -0800 Subject: [PATCH 13/17] New approach mixing in reasoning into original NL, still evaluating results but not amazing .. segmenting program into multiple chunks still wip locally --- src/scenicNL/adapters/anthropic_adapter.py | 201 ++++++++++++++++++++- src/scenicNL/adapters/prompts/tot_nl.txt | 12 ++ src/scenicNL/common.py | 14 ++ 3 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 src/scenicNL/adapters/prompts/tot_nl.txt diff --git a/src/scenicNL/adapters/anthropic_adapter.py b/src/scenicNL/adapters/anthropic_adapter.py index 38ae974..52c6bbd 100644 --- a/src/scenicNL/adapters/anthropic_adapter.py +++ b/src/scenicNL/adapters/anthropic_adapter.py @@ -8,7 +8,7 @@ from pathlib import Path from scenicNL.adapters.model_adapter import ModelAdapter from scenicNL.common import DISCUSSION_TEMPERATURE, NUM_EXPERTS, LLMPromptType, ModelInput, VectorDB, few_shot_prompt_with_rag, get_expert_synthesis_prompt -from scenicNL.common import get_discussion_prompt, get_discussion_to_program_prompt, format_scenic_tutorial_prompt, get_few_shot_ast_prompt +from scenicNL.common import get_discussion_prompt, get_discussion_to_program_prompt, format_scenic_tutorial_prompt, get_few_shot_ast_prompt, get_tot_nl_prompt import re import scenic import tempfile @@ -144,7 +144,63 @@ def _few_shot_reasoning_hyde( return prompt - + def _few_shot_reasoning_nl( + self, + model_input: ModelInput, + verbose: bool, + top_k: int = 3, + ) -> str: + if model_input.first_attempt_scenic_program is None: + if verbose: + print(f"Anthropic Model {self._model.value}\n" + f"_few_shot_reasoning_split: no first attempt scenic program, using original examples\n") + return self._few_shot_prompt(model_input=model_input, verbose=verbose) + + examples = self.index.query(model_input.first_attempt_scenic_program, top_k=top_k) + if examples is None: # if the query fails, we will use the original examples + if verbose: + print(f"Anthropic Model {self._model.value}\n" + f"_few_shot_reasoning_hyde: query into index for HyDE failed, using original examples\n") + else: + if verbose: + print(f"Anthropic Model {self._model.value}\n" + f"_few_shot_reasoning_hyde: query into index for HyDE successful, using examples {examples}\n") + + examples = model_input.examples # TODO: ignore HyDE for now + + relevant_model_input = ModelInput( + examples=[example for example in examples], + nat_lang_scene_des=model_input.nat_lang_scene_des, + first_attempt_scenic_program=model_input.first_attempt_scenic_program, + expert_discussion=model_input.expert_discussion, + ) + + prompt = get_discussion_to_program_prompt() + + task_and_others = prompt.split("{natural_language_description}") + task = task_and_others[0] + others = task_and_others[1].split("{example_1}") + example_1 = others[0] + others = others[1].split("{example_2}") + example_2 = others[0] + others = others[1].split("{example_3}") + example_3 = others[0] + + prompt = ( + f"{HUMAN_PROMPT}\n" + f"{task}{model_input.nat_lang_scene_des}\n" + f"{example_1}{relevant_model_input.examples[0]}\n" + f"{example_2}{relevant_model_input.examples[1]}\n" + f"{example_3}{relevant_model_input.examples[2]}\n" + f"{AI_PROMPT}" + ) + + if verbose: + print(f"Anthropic Model {self._model.value}\n" + f"_few_shot_reasoning_split: {prompt}") + + return prompt + def _few_shot_prompt_with_hyde( self, model_input: ModelInput, @@ -299,6 +355,10 @@ def _format_message( msg = self._few_shot_prompt_with_hyde(model_input=model_input, verbose=verbose) elif prompt_type == LLMPromptType.PREDICT_TOT_THEN_HYDE: msg = self._few_shot_reasoning_hyde(model_input=model_input, verbose=verbose) + elif prompt_type == LLMPromptType.PREDICT_TOT_THEN_SPLIT: + msg = self._few_shot_reasoning_split(model_input=model_input, verbose=verbose) + elif prompt_type == LLMPromptType.PREDICT_TOT_INTO_NL: + msg = self._few_shot_reasoning_nl(model_input=model_input, verbose=verbose) elif prompt_type == LLMPromptType.EXPERT_SYNTHESIS: msg = self._format_expert_synthesis_prompt(model_input=model_input, verbose=verbose) elif prompt_type == LLMPromptType.AST_FEEDBACK: @@ -408,6 +468,143 @@ def _predict( max_tokens_to_sample=max_length_tokens, model=self._model.value, ) + elif prompt_type == LLMPromptType.PREDICT_TOT_THEN_SPLIT: + # 1. Use tree of thought to answer all questions in the prompt + panel_answers = [] + for _ in range(NUM_EXPERTS): + claude_response = claude.completions.create( + prompt=self._format_message(model_input=model_input, prompt_type=LLMPromptType.EXPERT_DISCUSSION, verbose=verbose), + temperature=DISCUSSION_TEMPERATURE, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + panel_answers.append(claude_response.completion) + + if len(panel_answers) != NUM_EXPERTS: + raise ValueError(f"Expected {NUM_EXPERTS} panel answers, but got {len(panel_answers)}") + + model_input = ModelInput( + examples=model_input.examples, + nat_lang_scene_des=model_input.nat_lang_scene_des, + first_attempt_scenic_program=model_input.first_attempt_scenic_program, + panel_discussion=panel_answers, + expert_discussion=None # + ) + if verbose: + print(f"Anthropic Model {self._model.value}\n" + f"Tree of Thought: {panel_answers}\n") + + # 2. Ask an expert to synthesize the answers into a single answer + expert_response = claude.completions.create( + prompt=self._format_message(model_input=model_input, prompt_type=LLMPromptType.EXPERT_SYNTHESIS, verbose=verbose), + temperature=DISCUSSION_TEMPERATURE, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + expert_synthesis = expert_response.completion + + if verbose: + print(f"Anthropic Model {self._model.value}\n" + f"Expert Synthesis: {expert_synthesis}\n") + + # 3. Do a few shot predict on the natural language description + claude_response = claude.completions.create( + prompt=self._format_message(model_input=model_input, prompt_type=LLMPromptType.PREDICT_FEW_SHOT, verbose=verbose), + temperature=temperature, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + + # 4. Use the resulting program to query the index to do HyDE thus obtaining the top k programs + # We need to call Claude again + new_model_input = ModelInput( + examples=model_input.examples, # this will get overwritten by the search query + nat_lang_scene_des=model_input.nat_lang_scene_des, + first_attempt_scenic_program=claude_response.completion, # this is ONLY used for the query search in HyDE/RAG + expert_discussion=expert_synthesis, + panel_discussion=panel_answers + ) + + # 5. Use the top k programs as examples for the few shot prediction along with the answer from the tree of thought + claude_response = claude.completions.create( + prompt=self._format_message(model_input=new_model_input, prompt_type=prompt_type, verbose=verbose), + temperature=temperature, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + elif prompt_type == LLMPromptType.PREDICT_TOT_INTO_NL: + # 1. Use tree of thought to answer all questions in the prompt + panel_answers = [] + for _ in range(NUM_EXPERTS): + claude_response = claude.completions.create( + prompt=self._format_message(model_input=model_input, prompt_type=LLMPromptType.EXPERT_DISCUSSION, verbose=verbose), + temperature=DISCUSSION_TEMPERATURE, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + panel_answers.append(claude_response.completion) + + if len(panel_answers) != NUM_EXPERTS: + raise ValueError(f"Expected {NUM_EXPERTS} panel answers, but got {len(panel_answers)}") + + model_input = ModelInput( + examples=model_input.examples, + nat_lang_scene_des=model_input.nat_lang_scene_des, + first_attempt_scenic_program=model_input.first_attempt_scenic_program, + panel_discussion=panel_answers, + expert_discussion=None # + ) + if verbose: + print(f"Anthropic Model {self._model.value}\n" + f"Tree of Thought: {panel_answers}\n") + + # 2. Ask an expert to synthesize the answers into a single answer + expert_response = claude.completions.create( + prompt=self._format_message(model_input=model_input, prompt_type=LLMPromptType.EXPERT_SYNTHESIS, verbose=verbose), + temperature=DISCUSSION_TEMPERATURE, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + expert_synthesis = expert_response.completion + + if verbose: + print(f"Anthropic Model {self._model.value}\n" + f"Expert Synthesis: {expert_synthesis}\n") + + # 3. Create a new model input + new_model_input = ModelInput( + examples=model_input.examples, # this will get overwritten by the search query + nat_lang_scene_des=model_input.nat_lang_scene_des, + first_attempt_scenic_program=claude_response.completion, # this is ONLY used for the query search in HyDE/RAG + expert_discussion=expert_synthesis, + panel_discussion=panel_answers + ) + + # 3. Add the expert_synthesis directly into the natural language description + claude_response = claude.completions.create( + prompt=self._format_message(model_input=new_model_input, prompt_type=LLMPromptType.PREDICT_TOT_INTO_NL, verbose=verbose), + temperature=temperature, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + + # 4. Final new model input + final_model_input = ModelInput( + examples=model_input.examples, # this will get overwritten by the search query + nat_lang_scene_des=model_input.nat_lang_scene_des, + first_attempt_scenic_program=claude_response.completion, # this is ONLY used for the query search in HyDE/RAG + # expert_discussion=expert_synthesis, + # panel_discussion=panel_answers + ) + + + # 4. Do a few shot predict on the natural language description + claude_response = claude.completions.create( + prompt=self._format_message(model_input=final_model_input, prompt_type=LLMPromptType.PREDICT_FEW_SHOT, verbose=verbose), + temperature=temperature, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) else: claude_response = claude.completions.create( prompt=self._format_message(model_input=model_input, prompt_type=prompt_type, verbose=verbose), diff --git a/src/scenicNL/adapters/prompts/tot_nl.txt b/src/scenicNL/adapters/prompts/tot_nl.txt new file mode 100644 index 0000000..9dc47cf --- /dev/null +++ b/src/scenicNL/adapters/prompts/tot_nl.txt @@ -0,0 +1,12 @@ +Scenic is a probabilistic programming language for modeling the environments of autonomous cars. A Scenic program defines a distribution over scenes, configurations of physical objects and agents. Scenic can also define (probabilistic) policies for dynamic agents, allowing modeling scenarios where agents take actions over time in response to the state of the world. We use CARLA to render the scenes and simulate the agents. + +Let's start working towards having you help design a Scenic program by modifying the following natural language description: +{natural_language_description} + +Using the expert and panel reasoning below, please update the previous natural language description to include all the key information and Scenic constructs from the natural language description. + +Expert and panel discussions attached below. +{expert_discussion} + +{panel_discussion} + diff --git a/src/scenicNL/common.py b/src/scenicNL/common.py index 158c11b..e4d06b1 100644 --- a/src/scenicNL/common.py +++ b/src/scenicNL/common.py @@ -25,6 +25,8 @@ class LLMPromptType(Enum): PREDICT_FEW_SHOT_WITH_HYDE = "predict_few_shot_hyde" PREDICT_FEW_SHOT_WITH_HYDE_TOT = "predict_few_shot_hyde_tot" PREDICT_TOT_THEN_HYDE = "predict_tot_then_hyde" + PREDICT_TOT_THEN_SPLIT = "predict_tot_then_split" + PREDICT_TOT_INTO_NL = "predict_tot_into_nl" EXPERT_DISCUSSION = "expert_discussion" EXPERT_SYNTHESIS = "expert_synthesis" AST_FEEDBACK = "ast_feedback" @@ -39,6 +41,8 @@ class PromptFiles(Enum): TOT_EXPERT_DISCUSSION = os.path.join(PROMPT_PATH, 'tot_questions.txt') EXPERT_SYNTHESIS = os.path.join(PROMPT_PATH, 'expert_synthesis.txt') AST_FEEDBACK_CLAUDE = os.path.join(PROMPT_PATH, 'few_shot_ast.txt') + TOT_SPLIT = os.path.join(PROMPT_PATH, 'tot_split.txt') + TOT_NL = os.path.join(PROMPT_PATH, 'tot_nl.txt') @dataclass(frozen=True) class ModelInput: @@ -178,6 +182,16 @@ def get_few_shot_ast_prompt(model_input) -> str: return prompt +def get_tot_nl_prompt(model_input) -> str: + prompt = "" + with open(PromptFiles.TOT_NL.value) as f: + prompt = f.read() + prompt = prompt.format( + natural_language_description=model_input.nat_lang_scene_des, + expert_discussion=model_input.expert_discussion, + panel_discussion=model_input.panel_discussion + ) + return prompt class VectorDB(): def __init__( From c5b11a0443feab6be4cc37a1f6878e0a4c8266f3 Mon Sep 17 00:00:00 2001 From: Devan Shanker Date: Tue, 30 Jan 2024 00:45:10 -0800 Subject: [PATCH 14/17] Not the latest need to make a fix --- src/scenicNL/adapters/anthropic_adapter.py | 125 ++++++++++++---- src/scenicNL/adapters/model_adapter.py | 5 + .../prompts/scenic_tutorial_prompt.txt | 2 +- src/scenicNL/adapters/prompts/tot_split.txt | 141 ++++++++++++++++++ src/scenicNL/common.py | 6 + src/scenicNL/main.py | 8 + 6 files changed, 255 insertions(+), 32 deletions(-) create mode 100644 src/scenicNL/adapters/prompts/tot_split.txt diff --git a/src/scenicNL/adapters/anthropic_adapter.py b/src/scenicNL/adapters/anthropic_adapter.py index 52c6bbd..86cee4e 100644 --- a/src/scenicNL/adapters/anthropic_adapter.py +++ b/src/scenicNL/adapters/anthropic_adapter.py @@ -8,7 +8,7 @@ from pathlib import Path from scenicNL.adapters.model_adapter import ModelAdapter from scenicNL.common import DISCUSSION_TEMPERATURE, NUM_EXPERTS, LLMPromptType, ModelInput, VectorDB, few_shot_prompt_with_rag, get_expert_synthesis_prompt -from scenicNL.common import get_discussion_prompt, get_discussion_to_program_prompt, format_scenic_tutorial_prompt, get_few_shot_ast_prompt, get_tot_nl_prompt +from scenicNL.common import get_discussion_prompt, get_discussion_to_program_prompt, format_scenic_tutorial_prompt, get_few_shot_ast_prompt, get_tot_nl_prompt, get_discussion_to_split_prompt import re import scenic import tempfile @@ -192,6 +192,7 @@ def _few_shot_reasoning_nl( f"{example_1}{relevant_model_input.examples[0]}\n" f"{example_2}{relevant_model_input.examples[1]}\n" f"{example_3}{relevant_model_input.examples[2]}\n" + f"\nPlease start your answer with the character #" f"{AI_PROMPT}" ) @@ -201,6 +202,26 @@ def _few_shot_reasoning_nl( return prompt + def _few_shot_reasoning_split( + self, + model_input: ModelInput, + verbose: bool, + top_k: int = 3, + ) -> str: + example_1 = model_input.examples[0] + example_2 = model_input.examples[1] + example_3 = model_input.examples[2] + prompt = ( + f"{HUMAN_PROMPT}\n" + f"For the following Scenic program, write just the parameter definition section for the input below.{model_input.nat_lang_scene_des}\n" + f"{example_1}{model_input.examples[0]}\n" + f"{example_2}{model_input.examples[1]}\n" + f"{example_3}{model_input.examples[2]}\n" + f"{AI_PROMPT}" + ) + + return prompt + def _few_shot_prompt_with_hyde( self, model_input: ModelInput, @@ -273,7 +294,9 @@ def _zero_shot_prompt( f"_zero_shot_prompt: {prompt}") return prompt - + + + def _format_discussion_prompt( self, model_input: ModelInput, @@ -379,9 +402,9 @@ def _predict( max_length_tokens: int, prompt_type: LLMPromptType, verbose: bool, - max_retries: int + max_retries: int, + verbose_retries: bool, ) -> str: - verbose_retry = True # to prevent misuse of file handlers limits = httpx.Limits(max_keepalive_connections=1, max_connections=1) with Anthropic(connection_pool_limits=limits, max_retries=10) as claude: @@ -506,32 +529,74 @@ def _predict( if verbose: print(f"Anthropic Model {self._model.value}\n" f"Expert Synthesis: {expert_synthesis}\n") + print(f"Anthropic Model {self._model.value}\n" + f"Expert Synthesis: {expert_synthesis}\n") + reasoning = expert_synthesis + print(reasoning) + reasoning_list = reasoning.split(r'\n\d\.') + for item in reasoning_list: + print('>>', item[:20]) + + # self._few_shot_reasoning_split(model_input=model_input, verbose=False, top_k=3, reasoning=expert_synthesis) + incremental_scenic_program = '# Start of scenic program' + + split_prompt = get_discussion_to_split_prompt() + split_prompt = split_prompt.split('\n**\n') # 3. Do a few shot predict on the natural language description + for rest_prompt in split_prompt[1:]: + head_prompt = split_prompt[0].format( + example_1=model_input.examples[0], + natural_language_description=model_input.nat_lang_scene_des, + partial_scenic_program=incremental_scenic_program) + combo_prompt = head_prompt + rest_prompt + + combo_prompt = ( + f"{HUMAN_PROMPT}\n" + f"{combo_prompt}" + f"{AI_PROMPT}" + ) + + claude_response = claude.completions.create( + prompt=combo_prompt, + temperature=temperature, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + incremental_scenic_program += claude_response.completion + + stitch_prompt = ( + f"{HUMAN_PROMPT}\n" + f"Please connect together the following program snippets by deleting any redundant lines of text. You CANNOT make any other changes. The final output will be executed directly so please do not output any leading or trailing text. Thanks honey.\n" + f"{incremental_scenic_program}" + f"{AI_PROMPT}" + ) + claude_response = claude.completions.create( - prompt=self._format_message(model_input=model_input, prompt_type=LLMPromptType.PREDICT_FEW_SHOT, verbose=verbose), + prompt=stitch_prompt, temperature=temperature, max_tokens_to_sample=max_length_tokens, model=self._model.value, ) - + print('$$$$') + print(claude_response.completion) # 4. Use the resulting program to query the index to do HyDE thus obtaining the top k programs # We need to call Claude again - new_model_input = ModelInput( - examples=model_input.examples, # this will get overwritten by the search query - nat_lang_scene_des=model_input.nat_lang_scene_des, - first_attempt_scenic_program=claude_response.completion, # this is ONLY used for the query search in HyDE/RAG - expert_discussion=expert_synthesis, - panel_discussion=panel_answers - ) + # new_model_input = ModelInput( + # examples=model_input.examples, # this will get overwritten by the search query + # nat_lang_scene_des=model_input.nat_lang_scene_des, + # first_attempt_scenic_program=claude_response.completion, # this is ONLY used for the query search in HyDE/RAG + # expert_discussion=expert_synthesis, + # panel_discussion=panel_answers + # ) # 5. Use the top k programs as examples for the few shot prediction along with the answer from the tree of thought - claude_response = claude.completions.create( - prompt=self._format_message(model_input=new_model_input, prompt_type=prompt_type, verbose=verbose), - temperature=temperature, - max_tokens_to_sample=max_length_tokens, - model=self._model.value, - ) + # claude_response = claude.completions.create( + # prompt=self._format_message(model_input=new_model_input, prompt_type=prompt_type, verbose=verbose), + # temperature=temperature, + # max_tokens_to_sample=max_length_tokens, + # model=self._model.value, + # ) elif prompt_type == LLMPromptType.PREDICT_TOT_INTO_NL: # 1. Use tree of thought to answer all questions in the prompt panel_answers = [] @@ -580,7 +645,8 @@ def _predict( panel_discussion=panel_answers ) - # 3. Add the expert_synthesis directly into the natural language description + # 3. Add the expert_synthesis directly into the natural language description + # USING PREDICT_TOT_INTO_NL PROMPT - needs fixes for chattiness claude_response = claude.completions.create( prompt=self._format_message(model_input=new_model_input, prompt_type=LLMPromptType.PREDICT_TOT_INTO_NL, verbose=verbose), temperature=temperature, @@ -591,14 +657,13 @@ def _predict( # 4. Final new model input final_model_input = ModelInput( examples=model_input.examples, # this will get overwritten by the search query - nat_lang_scene_des=model_input.nat_lang_scene_des, - first_attempt_scenic_program=claude_response.completion, # this is ONLY used for the query search in HyDE/RAG + nat_lang_scene_des=claude_response.completion, + # first_attempt_scenic_program=claude_response.completion, # this is ONLY used for the query search in HyDE/RAG # expert_discussion=expert_synthesis, # panel_discussion=panel_answers ) - - # 4. Do a few shot predict on the natural language description + # 5. Do a few shot predict on the natural language description claude_response = claude.completions.create( prompt=self._format_message(model_input=final_model_input, prompt_type=LLMPromptType.PREDICT_FEW_SHOT, verbose=verbose), temperature=temperature, @@ -618,12 +683,10 @@ def _predict( with tempfile.TemporaryDirectory(dir=os.curdir) as temp_dir: retries = max_retries retries_dir = os.path.join(temp_dir, 'temp_dir') - print(f'^^^: {temp_dir}') os.makedirs(retries_dir) with tempfile.NamedTemporaryFile(dir=retries_dir, delete=False, suffix='.scenic') as temp_file: fname = temp_file.name - print(f'$$$: {fname}') while retries: with open(fname, 'w') as f: @@ -634,16 +697,16 @@ def _predict( try: # ast = scenic.syntax.parser.parse_file(fname) scenario = scenic.scenarioFromFile(fname, mode2D=True) - if verbose_retry: print('No error!') + if verbose_retries: print('No error!') retries = 0 # If this statement is reached program worked -> terminates loop except Exception as e: - if verbose_retry: print(f'Retrying... {retries}') + if verbose_retries: print(f'Retrying... {retries}') try: error_message = f"Error details below..\nmessage: {str(e)}\ntext: {e.text}\nlineno: {e.lineno}\nend_lineno: {e.end_lineno}\noffset: {e.offset}\nend_offset: {e.end_offset}" - if verbose_retry: print(error_message) + if verbose_retries: print(error_message) except: error_message = f'Error details below..\nmessage: {str(e)}' - if verbose_retry: print(error_message) + if verbose_retries: print(error_message) # Constructing correcting claude call new_model_input = ModelInput( @@ -661,6 +724,6 @@ def _predict( ) model_result = str(claude_response.completion) retries -= 1 - if verbose_retry: print(model_result) + if verbose_retries: print(model_result) return model_result \ No newline at end of file diff --git a/src/scenicNL/adapters/model_adapter.py b/src/scenicNL/adapters/model_adapter.py index b9c69cf..34ba633 100644 --- a/src/scenicNL/adapters/model_adapter.py +++ b/src/scenicNL/adapters/model_adapter.py @@ -50,6 +50,7 @@ def _predict( prompt_type: LLMPromptType, verbose: bool, max_retries: int, + verbose_retries: bool, ) -> str: """Perform a single prediction. Must be overriden by subclasses.""" raise NotImplementedError @@ -83,6 +84,7 @@ def _batch_processor( ignore_cache: bool, verbose: bool, max_retries: int, + verbose_retries: bool, ) -> Callable[[ModelInput], list[str | APIError]]: """ Return a function that takes a list of model inputs and returns a @@ -117,6 +119,7 @@ def process_single( prompt_type=prompt_type, verbose=verbose, max_retries=max_retries, + verbose_retries=verbose_retries, ) if prompt_type.value == LLMPromptType.PREDICT_PYTHON_API.value: api_input = ModelInput(model_input.examples, prediction) @@ -150,6 +153,7 @@ def predict_batch( num_workers: int = 10, ignore_cache: bool = False, max_retries: int = 0, + verbose_retries: bool = False, ) -> Iterable[list[str | APIError]]: """ Given a stream of model inputs, return a stream of predictions. This @@ -173,6 +177,7 @@ def predict_batch( ignore_cache=ignore_cache, verbose=verbose, max_retries=max_retries, + verbose_retries=verbose_retries, ) with ThreadPool(num_workers) as pool: diff --git a/src/scenicNL/adapters/prompts/scenic_tutorial_prompt.txt b/src/scenicNL/adapters/prompts/scenic_tutorial_prompt.txt index 136e998..1557b0e 100644 --- a/src/scenicNL/adapters/prompts/scenic_tutorial_prompt.txt +++ b/src/scenicNL/adapters/prompts/scenic_tutorial_prompt.txt @@ -2,7 +2,7 @@ Here is a quick tutorial about the Scenic language. Scenic scripts are typically divided into three sections: parameter definitions, scene setup, and behaviors. 1. Parameter Definitions: - In the parameter definitions section, you handle imports and define any parameters your scenario will use. +In the parameter definitions section, you handle imports and define any parameters your scenario will use. A Scenic script begins with importing necessary libraries. The first lines could be: "param map = localPath('../../../assets/maps/CARLA/Town05.xodr') param carla_map = 'Town05' diff --git a/src/scenicNL/adapters/prompts/tot_split.txt b/src/scenicNL/adapters/prompts/tot_split.txt new file mode 100644 index 0000000..c439b3e --- /dev/null +++ b/src/scenicNL/adapters/prompts/tot_split.txt @@ -0,0 +1,141 @@ +Scenic is a probabilistic programming language for modeling the environments of autonomous cars. A Scenic program defines a distribution over scenes, configurations of physical objects and agents. Scenic can also define (probabilistic) policies for dynamic agents, allowing modeling scenarios where agents take actions over time in response to the state of the world. We use CARLA to render the scenes and simulate the agents. + +Here is one example of a Scenic program: +{example_1} + +Let's start working towards having you create your own Scenic program based on the following natural language description: +{natural_language_description} + +Continue writing the + +{partial_scenic_program} +# continue writing from here + +Given the partial Scenic program, please continue and write only the next section of the Scenic program. I will APPEND your output directly to my code.scenic file so do not add any extra text or whitespace. + +** + +1. Parameter Definitions: +In the parameter definitions section, you handle imports and define any parameters your scenario will use. +A Scenic script begins with importing necessary libraries. +The first lines could be: "param map = localPath('../../../assets/maps/CARLA/Town05.xodr') +param carla_map = 'Town05' +model scenic.simulators.carla.model" to import the simulator library. +The set of maps allowed is only of the format ../../../assets/maps/CARLA/Town01.xodr for Town01, Town02, Town03, Town04, Town05, Town06, Town07. +No other map can be accessed. Please do not access exact streets or lanes as a result. +Then define any scene parameters, for example: "speed = Range(15, 25)" defines a parameter speed with values ranging from 15 to 25. + +Start this section with ## PARAMETER DEFINITIONS + +** + +2. Scene Setup: +In the scene setup section, you describe the static aspects of the scenario. +For example, "lane = Uniform(*network.lanes) +spawnPt = new OrientedPoint on lane.centerline +obstacle = new Trash at spawnPt offset by Range(1, -1) @ 0" creates a Trash obstacle offset from the centerline of a random lane. + +Start this section with ## SCENE SETUP + +** + +3. Behaviors: +In the behavior section, you define the dynamic behaviors involved in the scenario. +Objects which can take actions over time are called agents. We specifiy their dynamic behavior using the built-in property, `behavior`. A behavior defines a sequence of actions for the agent to take, which need not be fixed but can be probabilistic and depend on the state of the agent or other objects. In Scenic, an action is an instantaneous operation executed by an agent, like setting the steering angle of a car or turning on its headlights. +To define a behavior, we write a function which runs over the course of the scenario, periodically issuing actions. Scenic uses a discrete notion of time, so at each time step the function specifies zero or more actions for the agent to take. The function can also access the current state of the agent and other objects in the scene, and use this information to determine which actions to take. + +Here is an example of a behavior where a car is waiting for an ego car to be a certain distance away before it starts moving: +behavior WaitUntilClose(threshold=15): + while (distance from self to ego) > threshold: + wait + do FollowLaneBehavior() +Here, we repeatedly query the distance from the agent running the behavior (self) to the ego car; as long as it is above a threshold, we wait, which means take no actions. Once the threshold is met, we start driving by invoking the built in, FollowLaneBehavior. Since FollowLaneBehavior runs forever, we will never return to the WaitUntilClose behavior. + +Unlike ordinary Scenic code, control flow constructs such as if and while are allowed to depend on random variables inside a behavior. Any distributions defined inside a behavior are sampled at simulation time, not during scene sampling. Consider the following behavior: +behavior Foo(): + threshold = Range(4, 7) + while True: + if self.distanceToClosest(Pedestrian) < threshold: + strength = TruncatedNormal(0.8, 0.02, 0.5, 1) + take SetBrakeAction(strength), SetThrottleAction(0) + else: + take SetThrottleAction(0.5), SetBrakeAction(0) +Here, the value of threshold is sampled only once, at the beginning of the scenario when the behavior starts running. The value strength, on the other hand, is sampled every time step when the car is braking we use a slightly different braking strength (0.8 on average, but with 0 mean Gaussian noise added with standard deviation 0.02, truncated to the range [0.5, 1]). The behavior also uses the built-in function distanceToClosest, which returns the distance to the closest object of a given type (here, Pedestrian). This function is evaluated at each time step, so the car will always brake if a pedestrian is within the threshold distance. + +Suppose we want a car that follows a lane, stopping whenever it encounters an obstacle. Scenic provides a concept of interrupts. +behavior FollowAvoidingObstacles(): + try: + do FollowLaneBehavior() + interrupt when self.distanceToClosest(Object) < 5: + take SetBrakeAction(1) +This try-interrupt statement has similar syntax to the Python try statement (and in fact allows except clauses just as in Python), and begins in the same way: at first, the code block after the try: (the body) is executed. At the start of every time step during its execution, the condition from each interrupt clause is checked; if any are true, execution of the body is suspended and we instead begin to execute the corresponding interrupt handler. In the example above, there is only one interrupt, which fires when we come within 5 meters of any object. When that happens, FollowLaneBehavior is paused and we instead apply full braking for one time step. In the next step, we will resume FollowLaneBehavior wherever it left off, unless we are still within 5 meters of an object, in which case the interrupt will fire again. +If there are multiple interrupt clauses, successive clauses take precedence over those which precede them. Furthermore, such higher-priority interrupts can fire even during the execution of an earlier interrupt handler. This makes it easy to model a hierarchy of behaviors with different priorities; for example, we could implement a car which drives along a lane, passing slow cars and avoiding collisions, along the following lines: +behavior Drive(): + try: + do FollowLaneBehavior() + interrupt when self.distanceToNextObstacle() < 20: + do PassingBehavior() + interrupt when self.timeToCollision() < 5: + do CollisionAvoidance() +Alternatively, we can use `until` to specify a condition that must be met before the behavior can complete. +behavior ApproachAndTurnLeft(): + do FollowLaneBehavior() until (distance from self to intersection) < 10 + do WaitForTrafficLightBehavior() + do TurnLeftBehavior() +Or we can do it by time: +behavior DriveForAWhile(): + do FollowLaneBehavior() for 30 seconds +The alternative form `do behavior for n steps` uses time steps instead of real simulation time. + +Start this section with ## BEHAVIORS + +** + +4. Assignments: +In the assignments section, you assign the behaviors to initialized vehicle, bicyclist, and pedestrian objects now that behaviors have been defined. + +If needed, you can use syntax such as "A can see B" to make sure agent B is visible to agent A. +You can also use syntax like "distance from A to B" to make sure the distance between A and B is within a certain range. +oncomingCar = new Car on leftLaneSec.centerline, + with behavior OncomingCarBehavior() + +ego = new Car at spawnPt, + with behavior EgoBehavior(leftLaneSec) + +After this, "ego = new Car following roadDirection from spawnPt for Range(-50, -30), + with blueprint EGO_MODEL, + with behavior EgoBehavior(EGO_SPEED)" defines a dynamic agent with this behavior and other properties. All scenes must have an ego vehicle. + +blockingCar = new Car following roadDirection from ego for BLOCKING_CAR_DIST, + with viewAngle 90 deg + +Start this section with ## ASSIGNMENTS + +** + +5. Constraints: +In the constraints section, you define require statements to enforce initial and termination conditions. +Just as you can declare spatial constraints on scenes using the require statement, you can also impose constraints on dynamic scenarios. For example, if we don’t want to generate any simulations where car1 and car2 are simultaneously visible from the ego car, we could write: +require always not ((ego can see car1) and (ego can see car2)) +Here, `always condition` is a linear temporal (LTL) operator which can only be used inside a requirement, and which evaluates to true if and only if the condition is true at every time step of the scenario. So if the condition above is ever false during a simulation, the requirement will be violated, causing Scenic to reject that simulation and sample a new one. Similarly, we can require that a condition hold at some time during the scenario using the eventually operator: +require eventually ego in intersection + +So, if you need to include to include temporal aspects in your behavior, you use a require statement like so to model that no pedestrian comes close to self until the ego does (after which we place no further restrictions) +behavior WaitUntilClose(threshold=15): + while distance from self to ego > threshold: + require self.distanceToClosest(Pedestrian) > threshold + wait + do FollowLaneBehavior() + +Temporal operators can be combined with Boolean operators in a manner similar to LTL to build up more complex requirements: +require (always car.speed < 30) implies (always distance to car > 10) +Or relational: +require car2 not in intersection until car1 in intersection +require eventually car2 in intersection + +Require and terminate statements can be used to enforce post-conditions that determine that certain events occured or terminate the program after certain conditions are met thus satisfying the requirements of the scenario. Any require or terminate statement should not be about time and should be about events in the scene. For example, to make sure an oncoming Car is at a visible section of the lane: +require blockingCar can see oncomingCar +require (distance from blockingCar to oncomingCar) < DIST_BTW_BLOCKING_ONCOMING_CARS +require (distance from blockingCar to intersection) > DIST_TO_INTERSECTION + +Start this section with ## CONSTRAINTS \ No newline at end of file diff --git a/src/scenicNL/common.py b/src/scenicNL/common.py index e4d06b1..9e9807b 100644 --- a/src/scenicNL/common.py +++ b/src/scenicNL/common.py @@ -165,6 +165,12 @@ def get_discussion_to_program_prompt() -> str: return prompt +def get_discussion_to_split_prompt() -> str: + prompt = "" + with open(PromptFiles.TOT_SPLIT.value) as f: + prompt = f.read() + return prompt + def get_few_shot_ast_prompt(model_input) -> str: prompt = "" with open(PromptFiles.AST_FEEDBACK_CLAUDE.value) as f: diff --git a/src/scenicNL/main.py b/src/scenicNL/main.py index 5970c4f..3552bdb 100644 --- a/src/scenicNL/main.py +++ b/src/scenicNL/main.py @@ -169,6 +169,12 @@ def main(): help="Maximum number of compiler-in-the-loop retries.", ) +@click.option( + "--verbose_retries", + is_flag=True, + help="Boolean condition to display or omit verbose retries." +) + def main( query_path: Path, output_path: Path, @@ -187,6 +193,7 @@ def main( keep_filename: bool, temperature: float, max_retries: int, + verbose_retries: bool, ) -> None: """ Generate simulator scenes from natural language descriptions. @@ -270,6 +277,7 @@ def main( num_workers=num_workers, ignore_cache=ignore_cache, max_retries=max_retries, + verbose_retries=verbose_retries, ) ): for attempt, output in enumerate(outputs): From 41482e00bd5c6596c6003104f8aace16c9bf3b38 Mon Sep 17 00:00:00 2001 From: Devan Shanker Date: Tue, 30 Jan 2024 14:21:50 -0800 Subject: [PATCH 15/17] Support for intelligent reasoning splitting, program composition from split reasoning, program combining - add intelligent program reduction/redundancy removal --- src/scenicNL/adapters/anthropic_adapter.py | 22 +++++++++++++++++++++ src/scenicNL/adapters/prompts/tot_split.txt | 14 ++++++++----- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/scenicNL/adapters/anthropic_adapter.py b/src/scenicNL/adapters/anthropic_adapter.py index 86cee4e..b460e6a 100644 --- a/src/scenicNL/adapters/anthropic_adapter.py +++ b/src/scenicNL/adapters/anthropic_adapter.py @@ -545,9 +545,31 @@ def _predict( # 3. Do a few shot predict on the natural language description for rest_prompt in split_prompt[1:]: + reasoning_prompt = "Please concisely output the expert discussion answers most relevant to the requested task of writing part of a Scenic Program. Include NO OTHER input or references to question numbers as we are reading this out loud to a big crowd." + reasoning_prompt = ( + f"{HUMAN_PROMPT}\n" + f"\n** Natural Language Scene Description **\n" + f"{model_input.nat_lang_scene_des}" + f"\n**Expert Discussion**\n" + f"{expert_synthesis}" + f"\n**Reasoning Prompt **\n" + f"{reasoning_prompt}" + f"" + f"{AI_PROMPT}" + ) + + reasoning_summary = claude.completions.create( + prompt=reasoning_prompt, + temperature=temperature, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + print(f'$$$$\n{reasoning_summary.completion}') + head_prompt = split_prompt[0].format( example_1=model_input.examples[0], natural_language_description=model_input.nat_lang_scene_des, + reasoning_summary=reasoning_summary.completion, partial_scenic_program=incremental_scenic_program) combo_prompt = head_prompt + rest_prompt diff --git a/src/scenicNL/adapters/prompts/tot_split.txt b/src/scenicNL/adapters/prompts/tot_split.txt index c439b3e..1b8a146 100644 --- a/src/scenicNL/adapters/prompts/tot_split.txt +++ b/src/scenicNL/adapters/prompts/tot_split.txt @@ -1,18 +1,22 @@ Scenic is a probabilistic programming language for modeling the environments of autonomous cars. A Scenic program defines a distribution over scenes, configurations of physical objects and agents. Scenic can also define (probabilistic) policies for dynamic agents, allowing modeling scenarios where agents take actions over time in response to the state of the world. We use CARLA to render the scenes and simulate the agents. -Here is one example of a Scenic program: +** Here is one example of a Scenic program ** {example_1} -Let's start working towards having you create your own Scenic program based on the following natural language description: +** Let's start working towards having you create your own Scenic program based on the following natural language description ** {natural_language_description} -Continue writing the +** From this natural language description a collection of excerpts have compiled the following reasoning information ** +{reasoning_summary} +** TASK ** +Continue writing the partial Scenic program below. +Given the partial Scenic program and the reasoning excerpt, please continue and write only the next section of the Scenic program. I will APPEND your output directly to my code.scenic file so do not add any extra text or whitespace. + +## PARTIAL SCENIC PROGRAM ## {partial_scenic_program} # continue writing from here -Given the partial Scenic program, please continue and write only the next section of the Scenic program. I will APPEND your output directly to my code.scenic file so do not add any extra text or whitespace. - ** 1. Parameter Definitions: From 41a5004b60a6a168e3e2a8a29c72331136fee775 Mon Sep 17 00:00:00 2001 From: Devan Shanker Date: Tue, 30 Jan 2024 16:03:04 -0800 Subject: [PATCH 16/17] Work in progress corrections are removing all useful code and combination occasionally has repeat sections -> need debug output --- predict_tot_then_split.txt | 541 ++++++++++++++++++ src/scenicNL/adapters/anthropic_adapter.py | 25 +- .../adapters/prompts/few_shot_ast.txt | 4 +- src/scenicNL/adapters/prompts/tot_split.txt | 4 +- src/scenicNL/main.py | 1 + 5 files changed, 566 insertions(+), 9 deletions(-) create mode 100644 predict_tot_then_split.txt diff --git a/predict_tot_then_split.txt b/predict_tot_then_split.txt new file mode 100644 index 0000000..416c455 --- /dev/null +++ b/predict_tot_then_split.txt @@ -0,0 +1,541 @@ +Loaded all queries +Loaded all examples +Anthropic Model claude-instant-1.2 +Expert Synthesis: susp cirnthesize the sportReports' answers to question 1 +# Based on the description, the main objects that need to be included in the scene are: +# - kb Cruise autonomous vehicle AlbertforcedCruise AV") +# - A Honda Accord + +All \, experts agreed that the main objects in the scene aretran Cruise AV and the Honda Accord. + + mic: Here are the synthesized answers to roomsensors' responses: + +# Synthesize the sport experts' answers to question 1 +# Based on the description, the main objects that need to be included in the scene are: +# - A Cruise autonomous vehicle ("Cruise AV") +# - A Honda Accord + +# Synthesize the sport experts' answers to question 2 +# The spatial relationships between the objects are: +# - The Cruise AV is initially traveling westbound in the left lane on Sacramento Street +# - The Honda Accord is initially traveling in the right lane behind the Cruise AV *(# - The Honda Accord maneuvered across the Cruise AV's lane into the parking lane to the left of the Cruise AVensing# - The Honda Accord then began to merge back into the Cruise AV's lane +# - Both objects are moving + +All three experts agreed on the key spatial relationships between the objects, }); consecutive events described in Expert Vari's answerSpider lifeless most complete description. + +OptionsMiddleware Synthesize the sport experts' answers to question 3 +# The events that happened in the scene and their corresponding LTL formulas are: +# - Philadelphia politiciansda increases speed +# - eventually honda in parkingLane +# - eventually honda in cruiseLane +# - Cruise AV brakes +# - eventually contactBetween(cruise, honda) +# - eventually not(hondaAtScene) + +Expert 3 provided the most complete description of events in formal LTL formulas. meressue dummyCollect the key events described pylab the three experts' answers. + +desk Synthesize the sport experts' answers to question 4 +# Details missing from the description include: +# - What model years the vehicles are +# - The speeds of the vehicles +# - The weather/lighting conditions retail coefficients# - Exact location on Sacramento Street beyond the cross streets providedmill +# - Distances between the vehicles + +All three experts agreed onzech�mptotic details missing from the original description. + +# Synthesize the sport experts' answers to question 5 embedded +# Reasonable probability distributions over missing values: +# - Vehicle model years: DiscreteRange(2000, 2022) +# - Vehicle speeds: TruncatedNormal(35 mph, 10 mph, 0, 70 mph)ISINGCamera ��� styles +# - Weather: Discrete(['Sunny', 'Cloudy', 'Rainy'], [0.6, 0.3, 0.1]) +# - Location: Range(Polk St, Van Ness Ave) +# - Distances: TruncatedNormal(10m, 5m, 0, 50m) + +Thismultiplyizes the distributionanes지 Medic the three experts' answers to + the reasonableم of values for the missing details. + +# Synthesize the sport experts' answers to question 6 +# Relevant behaviors for the Scenic program: +# - FollowLaneBehavior Germans +# - LaneChangeBehavior +# - AccelerateForwardBehavior +# - ConstantThrottleBehavior +# - DriveAvoidingCollisions + +All three experts agreed onRaster key behaviors as mari relevant in the Scenic program. + +# Synthesize the sport experts' answers to question 7 +# Maps that couldachuset: +# - Town03 (downtown urban area with intersections could match Sacramento Street) +# - Town05 (large multilane roads could match Sacramento Street) +# - Best match would be Town03 since it mentions features like Sacramento Street having intersections. + +This synthesizes the map crystal238]--[@ the three experts,Drcubevc the reasoning for Town03 being the best match. + +# Synthesize the sport experts' answers to question 8 +# Vehicles that most closely match: brigade +# - Cruise AV: Tesla Model 3 (description mentions it is autonomous so electric vehicle is reasonable) +# - Honda Accord: Honda Accord (explicitly mentioned in description) + + swimmer three experts agreed flood these vehicles m Haplays�Tom being the closest desper to those described. + +# Synthesize the sport experts' answers to question 9 +# Details are reasonable given the information provided in the description. + radiationAll three experts agreed that the level of detail provided was reasonable. + +Myה }); ***ische' answers aimed to capture the key ethnел regarded across their crystal, while includingencer substrateinline details like the use of LTL formulas and reasoning excluded. Please let me know if Germans part of the synthesis needs more explanation or improvement. + + susp cirnthesize the sportReports' answers to question 1 +# Based on the description, the main objects that need to be included in the scene are: +# - kb Cruise autonomous vehicle AlbertforcedCruise AV") +# - A Honda Accord + +All \, experts agreed that the main objects in the scene aretran Cruise AV and the Honda Accord. + + mic: Here are the synthesized answers to roomsensors' responses: + +# Synthesize the sport experts' answers to question 1 +# Based on the description, the main objects that need to be included in the scene are: +# - A Cruise autonomous vehicle ("Cruise AV") +# - A Honda Accord + +# Synthesize the sport experts' answers to question 2 +# The spatial relationships between the objects are: +# - The Cruise AV is initially traveling westbound in the left lane on Sacramento Street +# - The Honda Accord is initially traveling in the right lane behind the Cruise AV *(# - The Honda Accord maneuvered across the Cruise AV's lane into the parking lane to the left of the Cruise AVensing# - The Honda Accord then began to merge back into the Cruise AV's lane +# - Both objects are moving + +All three experts agreed on the key spatial relationships between the objects, }); consecutive events described in Expert Vari's answerSpider lifeless most complete description. + +OptionsMiddleware Synthesize the sport experts' answers to question 3 +# The events that happened in the scene and their corresponding LTL formulas are: +# - Philadelphia politiciansda increases speed +# - eventually honda in parkingLane +# - eventually honda in cruiseLane +# - Cruise AV brakes +# - eventually contactBetween(cruise, honda) +# - eventually not(hondaAtScene) + +Expert 3 provided the most complete description of events in formal LTL formulas. meressue dummyCollect the key events described pylab the three experts' answers. + +desk Synthesize the sport experts' answers to question 4 +# Details missing from the description include: +# - What model years the vehicles are +# - The speeds of the vehicles +# - The weather/lighting conditions retail coefficients# - Exact location on Sacramento Street beyond the cross streets providedmill +# - Distances between the vehicles + +All three experts agreed onzech�mptotic details missing from the original description. + +# Synthesize the sport experts' answers to question 5 embedded +# Reasonable probability distributions over missing values: +# - Vehicle model years: DiscreteRange(2000, 2022) +# - Vehicle speeds: TruncatedNormal(35 mph, 10 mph, 0, 70 mph)ISINGCamera ��� styles +# - Weather: Discrete(['Sunny', 'Cloudy', 'Rainy'], [0.6, 0.3, 0.1]) +# - Location: Range(Polk St, Van Ness Ave) +# - Distances: TruncatedNormal(10m, 5m, 0, 50m) + +Thismultiplyizes the distributionanes지 Medic the three experts' answers to + the reasonableم of values for the missing details. + +# Synthesize the sport experts' answers to question 6 +# Relevant behaviors for the Scenic program: +# - FollowLaneBehavior Germans +# - LaneChangeBehavior +# - AccelerateForwardBehavior +# - ConstantThrottleBehavior +# - DriveAvoidingCollisions + +All three experts agreed onRaster key behaviors as mari relevant in the Scenic program. + +# Synthesize the sport experts' answers to question 7 +# Maps that couldachuset: +# - Town03 (downtown urban area with intersections could match Sacramento Street) +# - Town05 (large multilane roads could match Sacramento Street) +# - Best match would be Town03 since it mentions features like Sacramento Street having intersections. + +This synthesizes the map crystal238]--[@ the three experts,Drcubevc the reasoning for Town03 being the best match. + +# Synthesize the sport experts' answers to question 8 +# Vehicles that most closely match: brigade +# - Cruise AV: Tesla Model 3 (description mentions it is autonomous so electric vehicle is reasonable) +# - Honda Accord: Honda Accord (explicitly mentioned in description) + + swimmer three experts agreed flood these vehicles m Haplays�Tom being the closest desper to those described. + +# Synthesize the sport experts' answers to question 9 +# Details are reasonable given the information provided in the description. + radiationAll three experts agreed that the level of detail provided was reasonable. + +Myה }); ***ische' answers aimed to capture the key ethnел regarded across their crystal, while includingencer substrateinline details like the use of LTL formulas and reasoning excluded. Please let me know if Germans part of the synthesis needs more explanation or improvement. +>> susp cirnthesize th +<<<< + Here is the relevant information for writing part of a Scenic Program for this scenario: + +The key behaviors include FollowLaneBehavior, LaneChangeBehavior, AccelerateForwardBehavior, ConstantThrottleBehavior, and DriveAvoidingCollisions. The best matching map would be Town03 as it mentions features like Sacramento Street having intersections. The vehicles that most closely match are a Tesla Model 3 for the Cruise AV since it is described as autonomous, and a Honda Accord for the vehicle explicitly mentioned in the description. Details around missing values like vehicle speeds and locations are reasonable to represent with probability distributions like TruncatedNormal and Discrete ranges based on the information provided. +>>>> +<<<< + Here are key details for writing part of a Scenic Program to simulate this scene: + +The program should include a Cruise AV traveling westbound in the left lane on Sacramento Street and a Honda Accord initially behind it in the right lane. The Honda Accord should accelerate and change lanes into the parking lane to the left of the AV in an attempt to pass. It should then begin to merge back into the AV's lane. + +Relevant behaviors include FollowLaneBehavior, LaneChangeBehavior, AccelerateForwardBehavior, and DriveAvoidingCollisions. Maps like Town03 downtown area could provide a suitable environment with multilane roads and intersections matching the description. Vehicles like a Tesla Model 3 and Honda Accord are reasonable matches to the autonomous vehicle and Accord described. +>>>> +<<<< + Here is the Scenic program code relevant to recreating the scene: + +add_vehicle(tesla_model_3, "CruiseAV") +add_vehicle(honda_accord, "Honda") + +follow_lane("CruiseAV", left_lane) +follow_lane("Honda", right_lane) + +constant_throttle("CruiseAV", 35) +accelerate_forward("Honda") + +lane_change("Honda", parking_lane) +lane_change("Honda", left_lane) + +drive_avoiding_collisions + +set_map(town03) + +This code defines the key vehicles, sets their initial lanes and behaviors, and selects the most suitable map to recreate the scene described in the prompt. +>>>> +<<<< + Here are the key behaviors and maps relevant for writing part of a Scenic Program to simulate this traffic scenario: + +The key behaviors include FollowLaneBehavior, LaneChangeBehavior, AccelerateForwardBehavior, ConstantThrottleBehavior, and DriveAvoidingCollisions. The best map match would be Town03, as it features streets like Sacramento Street that have intersections. The vehicles that most closely match are a Tesla Model 3 for the Cruise AV since it is described as autonomous, and a Honda Accord for the vehicle explicitly mentioned in the description. +>>>> +<<<< + Here are the key behaviors and maps from the expert discussion that would be most relevant for writing part of a Scenic Program for this scenario: + +The relevant behaviors are FollowLaneBehavior, LaneChangeBehavior, AccelerateForwardBehavior, ConstantThrottleBehavior, and DriveAvoidingCollisions. The maps that could be used are Town03 or Town05, but Town03 is likely the best match as it may contain features like intersections mentioned in the scenario taking place on Sacramento Street. The vehicles that most closely match are a Tesla Model 3 for the Cruise AV since it is described as autonomous, and a Honda Accord for the vehicle explicitly mentioned in the description. +>>>> +¡¡¡¡ +# Start of scenic program## 1. Parameter Definitions: ## PARAMETER DEFINITIONS +param map = localPath('../../../assets/maps/CARLA/Town03.xodr') +param carla_map = 'Town03' +model scenic.simulators.carla.model + +speed = Range(15, 25) +location = Discrete(['Polk Street', 'Van Ness Avenue']) + +2. Vehicle Definitions: +In the vehicle definitions section, define the vehicles that will be in the scene along with any behaviors. Start this section with ## VEHICLE DEFINITIONS. + +## PARTIAL SCENIC PROGRAM ## +# Start of scenic program + +## PARAMETER DEFINITIONS +param map = localPath('../../../assets/maps/CARLA/Town03.xodr') +param carla_map = 'Town03' +model scenic.simulators.carla.model + +speed = Range(15, 25) +location = Discrete(['Polk Street', 'Van Ness Avenue']) + +## VEHICLE DEFINITIONS## 2. Scene Setup: ## VEHICLE DEFINITIONS + +cruise_av = new TeslaModel3 at new OrientedPoint on network.lanes, + with behavior FollowLaneBehavior(lane), + with maxSpeed = Uniform(speed) + +honda_accord = new HondaAccord behind cruise_av, + with behavior LaneChangeBehavior(cruise_av.lane, cruise_av.lane.leftLane), + with maxSpeed = Uniform(speed + 5) + +2. Scene Setup: +In the scene setup section, you describe the static aspects of the scenario. +For example, "lane = Uniform(*network.lanes) +spawnPt = new OrientedPoint on lane.centerline +obstacle = new Trash at spawnPt offset by Range(1, -1) @ 0" creates a Trash obstacle offset from the centerline of a random lane. + +Start this section with ## SCENE SETUP## 3. Behavior Definitions: ## SCENE SETUP + +lane = Uniform(*network.lanes) +cruise_av_spawn_pt = new OrientedPoint on lane.centerline +cruise_av = new TeslaModel3 at cruise_av_spawn_pt + +honda_spawn_pt = new OrientedPoint behind cruise_av_spawn_pt +honda = new HondaAccord at honda_spawn_pt + +require cruise_av.location == 'Polk Street' +require honda.location == 'Polk Street' + +## BEHAVIORS + +behavior FollowLaneBehavior(): + do cruise_av.followLane(lane) + +behavior AttemptPassBehavior(): + do honda.followLane(lane) + do honda.laneChange(lane.leftLane) + interrupt when honda touches cruise_av + do honda.brake() + do honda.laneChange(lane) + +terminate when any vehicle leaves bounds## 4. Assignments: Here is the completed Scenic program based on the provided description and reasoning: + +## PARTIAL SCENIC PROGRAM ## +# Start of scenic program + +## PARAMETER DEFINITIONS +param map = localPath('../../../assets/maps/CARLA/Town03.xodr') +param carla_map = 'Town03' +model scenic.simulators.carla.model + +speed = Range(15, 25) +location = Discrete(['Polk Street', 'Van Ness Avenue']) + +## VEHICLE DEFINITIONS + +cruise_av = new TeslaModel3 at new OrientedPoint on network.lanes, + with behavior FollowLaneBehavior(lane), + with maxSpeed = Uniform(speed) + +honda_accord = new HondaAccord behind cruise_av, + with behavior LaneChangeBehavior(cruise_av.lane, cruise_av.lane.leftLane), + with maxSpeed = Uniform(speed + 5) + + +## SCENE SETUP + +lane = Uniform(*network.lanes) +cruise_av_spawn_pt = new OrientedPoint on lane.centerline +cruise_av = new TeslaModel3 at cruise_av_spawn_pt + +honda_spawn_pt = new OrientedPoint behind cruise_av_spawn_pt +honda = new HondaAccord at honda_spawn_pt + + +require cruise_av.location == 'Polk Street' +require honda.location == 'Polk Street' + + +## BEHAVIORS + +behavior FollowLaneBehavior(): + do cruise_av.followLane(lane) + +behavior AttemptPassBehavior(): + do honda.followLane(lane) + do honda.laneChange(lane.leftLane) + interrupt when honda touches cruise_av + do honda.brake() + do honda.laneChange(lane) + +terminate when any vehicle leaves bounds + + +## ASSIGNMENTS + +cruise_av.behavior = FollowLaneBehavior() +honda.behavior = AttemptPassBehavior() + +require honda can see cruise_av +require distance from honda to cruise_av <= 20## 5. Constraints Here is the completed constraints section: + +## CONSTRAINTS + +require cruise_av.location == 'Polk Street' +require honda.location == 'Polk Street' + +require honda can see cruise_av +require distance from honda to cruise_av <= 20 + +require eventually contact between honda and cruise_av + +require after contact: + cruise_av.damagedMirror == true + honda.location != cruise_av.location + +terminate when any vehicle leaves bounds or honda leaves scene +!!!! +(((( + # Start of scenic program + +## PARAMETER DEFINITIONS +param map = localPath('../../../assets/maps/CARLA/Town03.xodr') +param carla_map = 'Town03' +model scenic.simulators.carla.model + +speed = Range(15, 25) +location = Discrete(['Polk Street', 'Van Ness Avenue']) + + +## VEHICLE DEFINITIONS + +cruise_av = new TeslaModel3 at new OrientedPoint on network.lanes, + with behavior FollowLaneBehavior(lane), + with maxSpeed = Uniform(speed) + +honda_accord = new HondaAccord behind cruise_av, + with behavior LaneChangeBehavior(cruise_av.lane, cruise_av.lane.leftLane), + with maxSpeed = Uniform(speed + 5) + + + +## SCENE SETUP + +lane = Uniform(*network.lanes) +cruise_av_spawn_pt = new OrientedPoint on lane.centerline +cruise_av = new TeslaModel3 at cruise_av_spawn_pt + +honda_spawn_pt = new OrientedPoint behind cruise_av_spawn_pt +honda = new HondaAccord at honda_spawn_pt + + + require cruise_av.location == 'Polk Street' + require honda.location == 'Polk Street' + + +## BEHAVIORS + +behavior FollowLaneBehavior(): + do cruise_av.followLane(lane) + +behavior AttemptPassBehavior(): + do honda.followLane(lane) + do honda.laneChange(lane.leftLane) + interrupt when honda touches cruise_av + do honda.brake() + do honda.laneChange(lane) + +terminate when any vehicle leaves bounds + + +cruise_av.behavior = FollowLaneBehavior() +honda.behavior = AttemptPassBehavior() + + require honda can see cruise_av + require distance from honda to cruise_av <= 20 + + require eventually contact between honda and cruise_av + + require after contact: + cruise_av.damagedMirror == true + honda.location != cruise_av.location + + terminate when any vehicle leaves bounds or honda leaves scene +)))) +¿¿¿¿ + # Start of scenic program + +## PARAMETER DEFINITIONS +param map = localPath('../../../assets/maps/CARLA/Town03.xodr') +param carla_map = 'Town03' +model scenic.simulators.carla.model + +speed = Range(15, 25) +location = Discrete(['Polk Street', 'Van Ness Avenue']) + + +## VEHICLE DEFINITIONS + +cruise_av = new TeslaModel3 at new OrientedPoint on network.lanes, + with behavior FollowLaneBehavior(lane), + with maxSpeed = Uniform(speed) + +honda_accord = new HondaAccord behind cruise_av, + with behavior LaneChangeBehavior(cruise_av.lane, cruise_av.lane.leftLane), + with maxSpeed = Uniform(speed + 5) + + + +## SCENE SETUP + +lane = Uniform(*network.lanes) +cruise_av_spawn_pt = new OrientedPoint on lane.centerline +cruise_av = new TeslaModel3 at cruise_av_spawn_pt + +honda_spawn_pt = new OrientedPoint behind cruise_av_spawn_pt +honda = new HondaAccord at honda_spawn_pt + + + require cruise_av.location == 'Polk Street' + require honda.location == 'Polk Street' + + +## BEHAVIORS + +behavior FollowLaneBehavior(): + do cruise_av.followLane(lane) + +behavior AttemptPassBehavior(): + do honda.followLane(lane) + do honda.laneChange(lane.leftLane) + interrupt when honda touches cruise_av + do honda.brake() + do honda.laneChange(lane) + +terminate when any vehicle leaves bounds + + +cruise_av.behavior = FollowLaneBehavior() +honda.behavior = AttemptPassBehavior() + + require honda can see cruise_av + require distance from honda to cruise_av <= 20 + + require eventually contact between honda and cruise_av + + require after contact: + cruise_av.damagedMirror == true + honda.location != cruise_av.location + + terminate when any vehicle leaves bounds or honda leaves scene +???? +°°°° + ## CONSTANTS +EGO_MODEL = "vehicle.lincoln.mkz_2017" +EGO_SPEED = 10 +EGO_BRAKING_THRESHOLD = 12 + +LEAD_CAR_SPEED = 10 +LEADCAR_BRAKING_THRESHOLD = 10 + +BRAKE_ACTION = 1.0 + +## DEFINING BEHAVIORS +behavior EgoBehavior(speed=10): + pass + +behavior LeadingCarBehavior(speed=10): + pass + +## DEFINING SPATIAL RELATIONS +lane = Uniform(*network.lanes) +^^^^ +°°°° + ## CONSTANTS +EGO_MODEL = "vehicle.lincoln.mkz_2017" +EGO_SPEED = 10 +EGO_BRAKING_THRESHOLD = 12 + +LEAD_CAR_SPEED = 10 +LEADCAR_BRAKING_THRESHOLD = 10 + +BRAKE_ACTION = 1.0 +^^^^ +[[[[ + ## CONSTANTS +EGO_MODEL = "vehicle.lincoln.mkz_2017" +EGO_SPEED = 10 +EGO_BRAKING_THRESHOLD = 12 + +LEAD_CAR_SPEED = 10 +LEADCAR_BRAKING_THRESHOLD = 10 + +BRAKE_ACTION = 1.0 +]]]] +Compiled input 0 - cruise_121422-pdf.txt successfully: +Executed input 0 - cruise_121422-pdf.txt successfully: +Output path 0 - cruise_121422-pdf.txt at: outputs/predict_tot_then_split_claude-instant-1/0-0.scenic +---------------- + + +Compile success rate: 100.0% +Execute success rate: 100.0% +## API error rate ##: 0.0% +outputs/predict_tot_then_split_claude-instant-1/_metadata/metrics.txt diff --git a/src/scenicNL/adapters/anthropic_adapter.py b/src/scenicNL/adapters/anthropic_adapter.py index b460e6a..753be94 100644 --- a/src/scenicNL/adapters/anthropic_adapter.py +++ b/src/scenicNL/adapters/anthropic_adapter.py @@ -542,9 +542,10 @@ def _predict( split_prompt = get_discussion_to_split_prompt() split_prompt = split_prompt.split('\n**\n') + sections = ["## 1. Parameter Definitions:", "## 2. Scene Setup:", "## 3. Behavior Definitions:", "## 4. Assignments:", "## 5. Constraints"] # 3. Do a few shot predict on the natural language description - for rest_prompt in split_prompt[1:]: + for rest_index, rest_prompt in enumerate(split_prompt[1:]): reasoning_prompt = "Please concisely output the expert discussion answers most relevant to the requested task of writing part of a Scenic Program. Include NO OTHER input or references to question numbers as we are reading this out loud to a big crowd." reasoning_prompt = ( f"{HUMAN_PROMPT}\n" @@ -564,7 +565,7 @@ def _predict( max_tokens_to_sample=max_length_tokens, model=self._model.value, ) - print(f'$$$$\n{reasoning_summary.completion}') + print(f'<<<<\n{reasoning_summary.completion}\n>>>>') head_prompt = split_prompt[0].format( example_1=model_input.examples[0], @@ -585,12 +586,18 @@ def _predict( max_tokens_to_sample=max_length_tokens, model=self._model.value, ) + incremental_scenic_program += sections[rest_index] incremental_scenic_program += claude_response.completion + print('¡¡¡¡') + print(incremental_scenic_program) + print('!!!!') + stitch_prompt = ( f"{HUMAN_PROMPT}\n" - f"Please connect together the following program snippets by deleting any redundant lines of text. You CANNOT make any other changes. The final output will be executed directly so please do not output any leading or trailing text. Thanks honey.\n" - f"{incremental_scenic_program}" + # f"Please connect together the following program snippets by deleting any redundant lines of text. You CANNOT make any other changes. The final output will be executed directly so please do not output any leading or trailing text. I will be beheaded if any program segment is missing. Thanks honey.\n" + f"Please delete any lines of code that are redundant between sections. You CANNOT make any other changes. The final output will be executed directly so please do not output any leading or trailing text. I will be beheaded if any program segment is missing. Thanks honey love you.\n" + f"\n{incremental_scenic_program}\n" f"{AI_PROMPT}" ) @@ -600,8 +607,9 @@ def _predict( max_tokens_to_sample=max_length_tokens, model=self._model.value, ) - print('$$$$') + print('((((') print(claude_response.completion) + print('))))') # 4. Use the resulting program to query the index to do HyDE thus obtaining the top k programs # We need to call Claude again # new_model_input = ModelInput( @@ -701,6 +709,9 @@ def _predict( ) model_result = str(claude_response.completion) + print('¿¿¿¿') + print(model_result) + print('????') with tempfile.TemporaryDirectory(dir=os.curdir) as temp_dir: retries = max_retries @@ -745,7 +756,11 @@ def _predict( model=self._model.value, ) model_result = str(claude_response.completion) + print('°°°°') + print(model_result) + print('^^^^') retries -= 1 if verbose_retries: print(model_result) + print(f'[[[[\n{model_result}\n]]]]') return model_result \ No newline at end of file diff --git a/src/scenicNL/adapters/prompts/few_shot_ast.txt b/src/scenicNL/adapters/prompts/few_shot_ast.txt index cefd117..a1cd378 100644 --- a/src/scenicNL/adapters/prompts/few_shot_ast.txt +++ b/src/scenicNL/adapters/prompts/few_shot_ast.txt @@ -6,7 +6,7 @@ Scenic is a probabilistic programming language for modeling the environments of Here is one example of a fully compiling Scenic program: {example_1} -Create a fully compiling Scenic program that models the description based on: +Modify the Scenic program you wrote that models the description based on: 1. The following natural language description: {natural_language_description} @@ -17,7 +17,7 @@ Create a fully compiling Scenic program that models the description based on: 3. The first compiler error raised with the scenic program: {compiler_error} -Please output a modified version of scenic_program modified so the compiler error does not appear. +Please output a modified version of your scenic_program modified so the compiler error does not appear. OUTPUT NO OTHER LEADING OR TRAILING TEXT OR WHITESPACE BESIDES THE CORRECTED SCENIC PROGRAM. NO ONE CARES. Output only the natural language description of the scenario as a comment at the top of the file and your proposed Scenic program. DO NOT output any other text or whitespace so I can run your program and see the results. diff --git a/src/scenicNL/adapters/prompts/tot_split.txt b/src/scenicNL/adapters/prompts/tot_split.txt index 1b8a146..dafe377 100644 --- a/src/scenicNL/adapters/prompts/tot_split.txt +++ b/src/scenicNL/adapters/prompts/tot_split.txt @@ -43,8 +43,8 @@ Start this section with ## SCENE SETUP ** -3. Behaviors: -In the behavior section, you define the dynamic behaviors involved in the scenario. +3. Behavior Definitions: +In the behavior definition section, you define the dynamic behaviors involved in the scenario. Objects which can take actions over time are called agents. We specifiy their dynamic behavior using the built-in property, `behavior`. A behavior defines a sequence of actions for the agent to take, which need not be fixed but can be probabilistic and depend on the state of the agent or other objects. In Scenic, an action is an instantaneous operation executed by an agent, like setting the steering angle of a car or turning on its headlights. To define a behavior, we write a function which runs over the course of the scenario, periodically issuing actions. Scenic uses a discrete notion of time, so at each time step the function specifies zero or more actions for the agent to take. The function can also access the current state of the agent and other objects in the scene, and use this information to determine which actions to take. diff --git a/src/scenicNL/main.py b/src/scenicNL/main.py index 3552bdb..b8ca2d0 100644 --- a/src/scenicNL/main.py +++ b/src/scenicNL/main.py @@ -321,6 +321,7 @@ def main( execute_fail += 1 with open(scenic_error_path, 'a') as f: f.write(f'{index} - {fstub} execute error: {e}\n') + print(f'Output path {debug} at: {fname}') print('----------------\n\n') end_time = time.time() From b6132deebc72cd4a7d62db13015c08411686c34e Mon Sep 17 00:00:00 2001 From: Devan Shanker Date: Wed, 31 Jan 2024 19:44:25 -0800 Subject: [PATCH 17/17] Incorporated specific task description for reasoning --- src/scenicNL/adapters/anthropic_adapter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/scenicNL/adapters/anthropic_adapter.py b/src/scenicNL/adapters/anthropic_adapter.py index 753be94..91c419c 100644 --- a/src/scenicNL/adapters/anthropic_adapter.py +++ b/src/scenicNL/adapters/anthropic_adapter.py @@ -553,6 +553,8 @@ def _predict( f"{model_input.nat_lang_scene_des}" f"\n**Expert Discussion**\n" f"{expert_synthesis}" + f"\n**Task Description**\n" + f"{rest_prompt}" f"\n**Reasoning Prompt **\n" f"{reasoning_prompt}" f""