diff --git a/predict_tot_then_split.txt b/predict_tot_then_split.txt new file mode 100644 index 0000000..416c455 --- /dev/null +++ b/predict_tot_then_split.txt @@ -0,0 +1,541 @@ +Loaded all queries +Loaded all examples +Anthropic Model claude-instant-1.2 +Expert Synthesis: susp cirnthesize the sportReports' answers to question 1 +# Based on the description, the main objects that need to be included in the scene are: +# - kb Cruise autonomous vehicle AlbertforcedCruise AV") +# - A Honda Accord + +All \, experts agreed that the main objects in the scene aretran Cruise AV and the Honda Accord. + + mic: Here are the synthesized answers to roomsensors' responses: + +# Synthesize the sport experts' answers to question 1 +# Based on the description, the main objects that need to be included in the scene are: +# - A Cruise autonomous vehicle ("Cruise AV") +# - A Honda Accord + +# Synthesize the sport experts' answers to question 2 +# The spatial relationships between the objects are: +# - The Cruise AV is initially traveling westbound in the left lane on Sacramento Street +# - The Honda Accord is initially traveling in the right lane behind the Cruise AV *(# - The Honda Accord maneuvered across the Cruise AV's lane into the parking lane to the left of the Cruise AVensing# - The Honda Accord then began to merge back into the Cruise AV's lane +# - Both objects are moving + +All three experts agreed on the key spatial relationships between the objects, }); consecutive events described in Expert Vari's answerSpider lifeless most complete description. + +OptionsMiddleware Synthesize the sport experts' answers to question 3 +# The events that happened in the scene and their corresponding LTL formulas are: +# - Philadelphia politiciansda increases speed +# - eventually honda in parkingLane +# - eventually honda in cruiseLane +# - Cruise AV brakes +# - eventually contactBetween(cruise, honda) +# - eventually not(hondaAtScene) + +Expert 3 provided the most complete description of events in formal LTL formulas. meressue dummyCollect the key events described pylab the three experts' answers. + +desk Synthesize the sport experts' answers to question 4 +# Details missing from the description include: +# - What model years the vehicles are +# - The speeds of the vehicles +# - The weather/lighting conditions retail coefficients# - Exact location on Sacramento Street beyond the cross streets providedmill +# - Distances between the vehicles + +All three experts agreed onzech�mptotic details missing from the original description. + +# Synthesize the sport experts' answers to question 5 embedded +# Reasonable probability distributions over missing values: +# - Vehicle model years: DiscreteRange(2000, 2022) +# - Vehicle speeds: TruncatedNormal(35 mph, 10 mph, 0, 70 mph)ISINGCamera ��� styles +# - Weather: Discrete(['Sunny', 'Cloudy', 'Rainy'], [0.6, 0.3, 0.1]) +# - Location: Range(Polk St, Van Ness Ave) +# - Distances: TruncatedNormal(10m, 5m, 0, 50m) + +Thismultiplyizes the distributionanes지 Medic the three experts' answers to + the reasonableم of values for the missing details. + +# Synthesize the sport experts' answers to question 6 +# Relevant behaviors for the Scenic program: +# - FollowLaneBehavior Germans +# - LaneChangeBehavior +# - AccelerateForwardBehavior +# - ConstantThrottleBehavior +# - DriveAvoidingCollisions + +All three experts agreed onRaster key behaviors as mari relevant in the Scenic program. + +# Synthesize the sport experts' answers to question 7 +# Maps that couldachuset: +# - Town03 (downtown urban area with intersections could match Sacramento Street) +# - Town05 (large multilane roads could match Sacramento Street) +# - Best match would be Town03 since it mentions features like Sacramento Street having intersections. + +This synthesizes the map crystal238]--[@ the three experts,Drcubevc the reasoning for Town03 being the best match. + +# Synthesize the sport experts' answers to question 8 +# Vehicles that most closely match: brigade +# - Cruise AV: Tesla Model 3 (description mentions it is autonomous so electric vehicle is reasonable) +# - Honda Accord: Honda Accord (explicitly mentioned in description) + + swimmer three experts agreed flood these vehicles m Haplays�Tom being the closest desper to those described. + +# Synthesize the sport experts' answers to question 9 +# Details are reasonable given the information provided in the description. + radiationAll three experts agreed that the level of detail provided was reasonable. + +Myה }); ***ische' answers aimed to capture the key ethnел regarded across their crystal, while includingencer substrateinline details like the use of LTL formulas and reasoning excluded. Please let me know if Germans part of the synthesis needs more explanation or improvement. + + susp cirnthesize the sportReports' answers to question 1 +# Based on the description, the main objects that need to be included in the scene are: +# - kb Cruise autonomous vehicle AlbertforcedCruise AV") +# - A Honda Accord + +All \, experts agreed that the main objects in the scene aretran Cruise AV and the Honda Accord. + + mic: Here are the synthesized answers to roomsensors' responses: + +# Synthesize the sport experts' answers to question 1 +# Based on the description, the main objects that need to be included in the scene are: +# - A Cruise autonomous vehicle ("Cruise AV") +# - A Honda Accord + +# Synthesize the sport experts' answers to question 2 +# The spatial relationships between the objects are: +# - The Cruise AV is initially traveling westbound in the left lane on Sacramento Street +# - The Honda Accord is initially traveling in the right lane behind the Cruise AV *(# - The Honda Accord maneuvered across the Cruise AV's lane into the parking lane to the left of the Cruise AVensing# - The Honda Accord then began to merge back into the Cruise AV's lane +# - Both objects are moving + +All three experts agreed on the key spatial relationships between the objects, }); consecutive events described in Expert Vari's answerSpider lifeless most complete description. + +OptionsMiddleware Synthesize the sport experts' answers to question 3 +# The events that happened in the scene and their corresponding LTL formulas are: +# - Philadelphia politiciansda increases speed +# - eventually honda in parkingLane +# - eventually honda in cruiseLane +# - Cruise AV brakes +# - eventually contactBetween(cruise, honda) +# - eventually not(hondaAtScene) + +Expert 3 provided the most complete description of events in formal LTL formulas. meressue dummyCollect the key events described pylab the three experts' answers. + +desk Synthesize the sport experts' answers to question 4 +# Details missing from the description include: +# - What model years the vehicles are +# - The speeds of the vehicles +# - The weather/lighting conditions retail coefficients# - Exact location on Sacramento Street beyond the cross streets providedmill +# - Distances between the vehicles + +All three experts agreed onzech�mptotic details missing from the original description. + +# Synthesize the sport experts' answers to question 5 embedded +# Reasonable probability distributions over missing values: +# - Vehicle model years: DiscreteRange(2000, 2022) +# - Vehicle speeds: TruncatedNormal(35 mph, 10 mph, 0, 70 mph)ISINGCamera ��� styles +# - Weather: Discrete(['Sunny', 'Cloudy', 'Rainy'], [0.6, 0.3, 0.1]) +# - Location: Range(Polk St, Van Ness Ave) +# - Distances: TruncatedNormal(10m, 5m, 0, 50m) + +Thismultiplyizes the distributionanes지 Medic the three experts' answers to + the reasonableم of values for the missing details. + +# Synthesize the sport experts' answers to question 6 +# Relevant behaviors for the Scenic program: +# - FollowLaneBehavior Germans +# - LaneChangeBehavior +# - AccelerateForwardBehavior +# - ConstantThrottleBehavior +# - DriveAvoidingCollisions + +All three experts agreed onRaster key behaviors as mari relevant in the Scenic program. + +# Synthesize the sport experts' answers to question 7 +# Maps that couldachuset: +# - Town03 (downtown urban area with intersections could match Sacramento Street) +# - Town05 (large multilane roads could match Sacramento Street) +# - Best match would be Town03 since it mentions features like Sacramento Street having intersections. + +This synthesizes the map crystal238]--[@ the three experts,Drcubevc the reasoning for Town03 being the best match. + +# Synthesize the sport experts' answers to question 8 +# Vehicles that most closely match: brigade +# - Cruise AV: Tesla Model 3 (description mentions it is autonomous so electric vehicle is reasonable) +# - Honda Accord: Honda Accord (explicitly mentioned in description) + + swimmer three experts agreed flood these vehicles m Haplays�Tom being the closest desper to those described. + +# Synthesize the sport experts' answers to question 9 +# Details are reasonable given the information provided in the description. + radiationAll three experts agreed that the level of detail provided was reasonable. + +Myה }); ***ische' answers aimed to capture the key ethnел regarded across their crystal, while includingencer substrateinline details like the use of LTL formulas and reasoning excluded. Please let me know if Germans part of the synthesis needs more explanation or improvement. +>> susp cirnthesize th +<<<< + Here is the relevant information for writing part of a Scenic Program for this scenario: + +The key behaviors include FollowLaneBehavior, LaneChangeBehavior, AccelerateForwardBehavior, ConstantThrottleBehavior, and DriveAvoidingCollisions. The best matching map would be Town03 as it mentions features like Sacramento Street having intersections. The vehicles that most closely match are a Tesla Model 3 for the Cruise AV since it is described as autonomous, and a Honda Accord for the vehicle explicitly mentioned in the description. Details around missing values like vehicle speeds and locations are reasonable to represent with probability distributions like TruncatedNormal and Discrete ranges based on the information provided. +>>>> +<<<< + Here are key details for writing part of a Scenic Program to simulate this scene: + +The program should include a Cruise AV traveling westbound in the left lane on Sacramento Street and a Honda Accord initially behind it in the right lane. The Honda Accord should accelerate and change lanes into the parking lane to the left of the AV in an attempt to pass. It should then begin to merge back into the AV's lane. + +Relevant behaviors include FollowLaneBehavior, LaneChangeBehavior, AccelerateForwardBehavior, and DriveAvoidingCollisions. Maps like Town03 downtown area could provide a suitable environment with multilane roads and intersections matching the description. Vehicles like a Tesla Model 3 and Honda Accord are reasonable matches to the autonomous vehicle and Accord described. +>>>> +<<<< + Here is the Scenic program code relevant to recreating the scene: + +add_vehicle(tesla_model_3, "CruiseAV") +add_vehicle(honda_accord, "Honda") + +follow_lane("CruiseAV", left_lane) +follow_lane("Honda", right_lane) + +constant_throttle("CruiseAV", 35) +accelerate_forward("Honda") + +lane_change("Honda", parking_lane) +lane_change("Honda", left_lane) + +drive_avoiding_collisions + +set_map(town03) + +This code defines the key vehicles, sets their initial lanes and behaviors, and selects the most suitable map to recreate the scene described in the prompt. +>>>> +<<<< + Here are the key behaviors and maps relevant for writing part of a Scenic Program to simulate this traffic scenario: + +The key behaviors include FollowLaneBehavior, LaneChangeBehavior, AccelerateForwardBehavior, ConstantThrottleBehavior, and DriveAvoidingCollisions. The best map match would be Town03, as it features streets like Sacramento Street that have intersections. The vehicles that most closely match are a Tesla Model 3 for the Cruise AV since it is described as autonomous, and a Honda Accord for the vehicle explicitly mentioned in the description. +>>>> +<<<< + Here are the key behaviors and maps from the expert discussion that would be most relevant for writing part of a Scenic Program for this scenario: + +The relevant behaviors are FollowLaneBehavior, LaneChangeBehavior, AccelerateForwardBehavior, ConstantThrottleBehavior, and DriveAvoidingCollisions. The maps that could be used are Town03 or Town05, but Town03 is likely the best match as it may contain features like intersections mentioned in the scenario taking place on Sacramento Street. The vehicles that most closely match are a Tesla Model 3 for the Cruise AV since it is described as autonomous, and a Honda Accord for the vehicle explicitly mentioned in the description. +>>>> +¡¡¡¡ +# Start of scenic program## 1. Parameter Definitions: ## PARAMETER DEFINITIONS +param map = localPath('../../../assets/maps/CARLA/Town03.xodr') +param carla_map = 'Town03' +model scenic.simulators.carla.model + +speed = Range(15, 25) +location = Discrete(['Polk Street', 'Van Ness Avenue']) + +2. Vehicle Definitions: +In the vehicle definitions section, define the vehicles that will be in the scene along with any behaviors. Start this section with ## VEHICLE DEFINITIONS. + +## PARTIAL SCENIC PROGRAM ## +# Start of scenic program + +## PARAMETER DEFINITIONS +param map = localPath('../../../assets/maps/CARLA/Town03.xodr') +param carla_map = 'Town03' +model scenic.simulators.carla.model + +speed = Range(15, 25) +location = Discrete(['Polk Street', 'Van Ness Avenue']) + +## VEHICLE DEFINITIONS## 2. Scene Setup: ## VEHICLE DEFINITIONS + +cruise_av = new TeslaModel3 at new OrientedPoint on network.lanes, + with behavior FollowLaneBehavior(lane), + with maxSpeed = Uniform(speed) + +honda_accord = new HondaAccord behind cruise_av, + with behavior LaneChangeBehavior(cruise_av.lane, cruise_av.lane.leftLane), + with maxSpeed = Uniform(speed + 5) + +2. Scene Setup: +In the scene setup section, you describe the static aspects of the scenario. +For example, "lane = Uniform(*network.lanes) +spawnPt = new OrientedPoint on lane.centerline +obstacle = new Trash at spawnPt offset by Range(1, -1) @ 0" creates a Trash obstacle offset from the centerline of a random lane. + +Start this section with ## SCENE SETUP## 3. Behavior Definitions: ## SCENE SETUP + +lane = Uniform(*network.lanes) +cruise_av_spawn_pt = new OrientedPoint on lane.centerline +cruise_av = new TeslaModel3 at cruise_av_spawn_pt + +honda_spawn_pt = new OrientedPoint behind cruise_av_spawn_pt +honda = new HondaAccord at honda_spawn_pt + +require cruise_av.location == 'Polk Street' +require honda.location == 'Polk Street' + +## BEHAVIORS + +behavior FollowLaneBehavior(): + do cruise_av.followLane(lane) + +behavior AttemptPassBehavior(): + do honda.followLane(lane) + do honda.laneChange(lane.leftLane) + interrupt when honda touches cruise_av + do honda.brake() + do honda.laneChange(lane) + +terminate when any vehicle leaves bounds## 4. Assignments: Here is the completed Scenic program based on the provided description and reasoning: + +## PARTIAL SCENIC PROGRAM ## +# Start of scenic program + +## PARAMETER DEFINITIONS +param map = localPath('../../../assets/maps/CARLA/Town03.xodr') +param carla_map = 'Town03' +model scenic.simulators.carla.model + +speed = Range(15, 25) +location = Discrete(['Polk Street', 'Van Ness Avenue']) + +## VEHICLE DEFINITIONS + +cruise_av = new TeslaModel3 at new OrientedPoint on network.lanes, + with behavior FollowLaneBehavior(lane), + with maxSpeed = Uniform(speed) + +honda_accord = new HondaAccord behind cruise_av, + with behavior LaneChangeBehavior(cruise_av.lane, cruise_av.lane.leftLane), + with maxSpeed = Uniform(speed + 5) + + +## SCENE SETUP + +lane = Uniform(*network.lanes) +cruise_av_spawn_pt = new OrientedPoint on lane.centerline +cruise_av = new TeslaModel3 at cruise_av_spawn_pt + +honda_spawn_pt = new OrientedPoint behind cruise_av_spawn_pt +honda = new HondaAccord at honda_spawn_pt + + +require cruise_av.location == 'Polk Street' +require honda.location == 'Polk Street' + + +## BEHAVIORS + +behavior FollowLaneBehavior(): + do cruise_av.followLane(lane) + +behavior AttemptPassBehavior(): + do honda.followLane(lane) + do honda.laneChange(lane.leftLane) + interrupt when honda touches cruise_av + do honda.brake() + do honda.laneChange(lane) + +terminate when any vehicle leaves bounds + + +## ASSIGNMENTS + +cruise_av.behavior = FollowLaneBehavior() +honda.behavior = AttemptPassBehavior() + +require honda can see cruise_av +require distance from honda to cruise_av <= 20## 5. Constraints Here is the completed constraints section: + +## CONSTRAINTS + +require cruise_av.location == 'Polk Street' +require honda.location == 'Polk Street' + +require honda can see cruise_av +require distance from honda to cruise_av <= 20 + +require eventually contact between honda and cruise_av + +require after contact: + cruise_av.damagedMirror == true + honda.location != cruise_av.location + +terminate when any vehicle leaves bounds or honda leaves scene +!!!! +(((( + # Start of scenic program + +## PARAMETER DEFINITIONS +param map = localPath('../../../assets/maps/CARLA/Town03.xodr') +param carla_map = 'Town03' +model scenic.simulators.carla.model + +speed = Range(15, 25) +location = Discrete(['Polk Street', 'Van Ness Avenue']) + + +## VEHICLE DEFINITIONS + +cruise_av = new TeslaModel3 at new OrientedPoint on network.lanes, + with behavior FollowLaneBehavior(lane), + with maxSpeed = Uniform(speed) + +honda_accord = new HondaAccord behind cruise_av, + with behavior LaneChangeBehavior(cruise_av.lane, cruise_av.lane.leftLane), + with maxSpeed = Uniform(speed + 5) + + + +## SCENE SETUP + +lane = Uniform(*network.lanes) +cruise_av_spawn_pt = new OrientedPoint on lane.centerline +cruise_av = new TeslaModel3 at cruise_av_spawn_pt + +honda_spawn_pt = new OrientedPoint behind cruise_av_spawn_pt +honda = new HondaAccord at honda_spawn_pt + + + require cruise_av.location == 'Polk Street' + require honda.location == 'Polk Street' + + +## BEHAVIORS + +behavior FollowLaneBehavior(): + do cruise_av.followLane(lane) + +behavior AttemptPassBehavior(): + do honda.followLane(lane) + do honda.laneChange(lane.leftLane) + interrupt when honda touches cruise_av + do honda.brake() + do honda.laneChange(lane) + +terminate when any vehicle leaves bounds + + +cruise_av.behavior = FollowLaneBehavior() +honda.behavior = AttemptPassBehavior() + + require honda can see cruise_av + require distance from honda to cruise_av <= 20 + + require eventually contact between honda and cruise_av + + require after contact: + cruise_av.damagedMirror == true + honda.location != cruise_av.location + + terminate when any vehicle leaves bounds or honda leaves scene +)))) +¿¿¿¿ + # Start of scenic program + +## PARAMETER DEFINITIONS +param map = localPath('../../../assets/maps/CARLA/Town03.xodr') +param carla_map = 'Town03' +model scenic.simulators.carla.model + +speed = Range(15, 25) +location = Discrete(['Polk Street', 'Van Ness Avenue']) + + +## VEHICLE DEFINITIONS + +cruise_av = new TeslaModel3 at new OrientedPoint on network.lanes, + with behavior FollowLaneBehavior(lane), + with maxSpeed = Uniform(speed) + +honda_accord = new HondaAccord behind cruise_av, + with behavior LaneChangeBehavior(cruise_av.lane, cruise_av.lane.leftLane), + with maxSpeed = Uniform(speed + 5) + + + +## SCENE SETUP + +lane = Uniform(*network.lanes) +cruise_av_spawn_pt = new OrientedPoint on lane.centerline +cruise_av = new TeslaModel3 at cruise_av_spawn_pt + +honda_spawn_pt = new OrientedPoint behind cruise_av_spawn_pt +honda = new HondaAccord at honda_spawn_pt + + + require cruise_av.location == 'Polk Street' + require honda.location == 'Polk Street' + + +## BEHAVIORS + +behavior FollowLaneBehavior(): + do cruise_av.followLane(lane) + +behavior AttemptPassBehavior(): + do honda.followLane(lane) + do honda.laneChange(lane.leftLane) + interrupt when honda touches cruise_av + do honda.brake() + do honda.laneChange(lane) + +terminate when any vehicle leaves bounds + + +cruise_av.behavior = FollowLaneBehavior() +honda.behavior = AttemptPassBehavior() + + require honda can see cruise_av + require distance from honda to cruise_av <= 20 + + require eventually contact between honda and cruise_av + + require after contact: + cruise_av.damagedMirror == true + honda.location != cruise_av.location + + terminate when any vehicle leaves bounds or honda leaves scene +???? +°°°° + ## CONSTANTS +EGO_MODEL = "vehicle.lincoln.mkz_2017" +EGO_SPEED = 10 +EGO_BRAKING_THRESHOLD = 12 + +LEAD_CAR_SPEED = 10 +LEADCAR_BRAKING_THRESHOLD = 10 + +BRAKE_ACTION = 1.0 + +## DEFINING BEHAVIORS +behavior EgoBehavior(speed=10): + pass + +behavior LeadingCarBehavior(speed=10): + pass + +## DEFINING SPATIAL RELATIONS +lane = Uniform(*network.lanes) +^^^^ +°°°° + ## CONSTANTS +EGO_MODEL = "vehicle.lincoln.mkz_2017" +EGO_SPEED = 10 +EGO_BRAKING_THRESHOLD = 12 + +LEAD_CAR_SPEED = 10 +LEADCAR_BRAKING_THRESHOLD = 10 + +BRAKE_ACTION = 1.0 +^^^^ +[[[[ + ## CONSTANTS +EGO_MODEL = "vehicle.lincoln.mkz_2017" +EGO_SPEED = 10 +EGO_BRAKING_THRESHOLD = 12 + +LEAD_CAR_SPEED = 10 +LEADCAR_BRAKING_THRESHOLD = 10 + +BRAKE_ACTION = 1.0 +]]]] +Compiled input 0 - cruise_121422-pdf.txt successfully: +Executed input 0 - cruise_121422-pdf.txt successfully: +Output path 0 - cruise_121422-pdf.txt at: outputs/predict_tot_then_split_claude-instant-1/0-0.scenic +---------------- + + +Compile success rate: 100.0% +Execute success rate: 100.0% +## API error rate ##: 0.0% +outputs/predict_tot_then_split_claude-instant-1/_metadata/metrics.txt diff --git a/src/scenicNL/adapters/anthropic_adapter.py b/src/scenicNL/adapters/anthropic_adapter.py index dce8947..91c419c 100644 --- a/src/scenicNL/adapters/anthropic_adapter.py +++ b/src/scenicNL/adapters/anthropic_adapter.py @@ -4,10 +4,14 @@ from anthropic import Anthropic, AI_PROMPT, HUMAN_PROMPT import httpx +import os +from pathlib import Path from scenicNL.adapters.model_adapter import ModelAdapter from scenicNL.common import DISCUSSION_TEMPERATURE, NUM_EXPERTS, LLMPromptType, ModelInput, VectorDB, few_shot_prompt_with_rag, get_expert_synthesis_prompt -from scenicNL.common import get_discussion_prompt, get_discussion_to_program_prompt, format_scenic_tutorial_prompt - +from scenicNL.common import get_discussion_prompt, get_discussion_to_program_prompt, format_scenic_tutorial_prompt, get_few_shot_ast_prompt, get_tot_nl_prompt, get_discussion_to_split_prompt +import re +import scenic +import tempfile class AnthropicModel(Enum): CLAUDE_INSTANT = "claude-instant-1.2" @@ -51,6 +55,7 @@ def _few_shot_prompt( f"{format_scenic_tutorial_prompt()}\n\n" f"Here is the natural language description from the user: \n" f"\n\n{model_input.nat_lang_scene_des}\n\n" + f"\n\nWrite a scenic program that models the natural language description. Provide NO additional commentary before or after. Only output the code." f"\n\n{AI_PROMPT}" ) @@ -139,7 +144,84 @@ def _few_shot_reasoning_hyde( return prompt - + def _few_shot_reasoning_nl( + self, + model_input: ModelInput, + verbose: bool, + top_k: int = 3, + ) -> str: + if model_input.first_attempt_scenic_program is None: + if verbose: + print(f"Anthropic Model {self._model.value}\n" + f"_few_shot_reasoning_split: no first attempt scenic program, using original examples\n") + return self._few_shot_prompt(model_input=model_input, verbose=verbose) + + examples = self.index.query(model_input.first_attempt_scenic_program, top_k=top_k) + if examples is None: # if the query fails, we will use the original examples + if verbose: + print(f"Anthropic Model {self._model.value}\n" + f"_few_shot_reasoning_hyde: query into index for HyDE failed, using original examples\n") + else: + if verbose: + print(f"Anthropic Model {self._model.value}\n" + f"_few_shot_reasoning_hyde: query into index for HyDE successful, using examples {examples}\n") + + examples = model_input.examples # TODO: ignore HyDE for now + + relevant_model_input = ModelInput( + examples=[example for example in examples], + nat_lang_scene_des=model_input.nat_lang_scene_des, + first_attempt_scenic_program=model_input.first_attempt_scenic_program, + expert_discussion=model_input.expert_discussion, + ) + + prompt = get_discussion_to_program_prompt() + + task_and_others = prompt.split("{natural_language_description}") + task = task_and_others[0] + others = task_and_others[1].split("{example_1}") + example_1 = others[0] + others = others[1].split("{example_2}") + example_2 = others[0] + others = others[1].split("{example_3}") + example_3 = others[0] + + prompt = ( + f"{HUMAN_PROMPT}\n" + f"{task}{model_input.nat_lang_scene_des}\n" + f"{example_1}{relevant_model_input.examples[0]}\n" + f"{example_2}{relevant_model_input.examples[1]}\n" + f"{example_3}{relevant_model_input.examples[2]}\n" + f"\nPlease start your answer with the character #" + f"{AI_PROMPT}" + ) + + if verbose: + print(f"Anthropic Model {self._model.value}\n" + f"_few_shot_reasoning_split: {prompt}") + + return prompt + + def _few_shot_reasoning_split( + self, + model_input: ModelInput, + verbose: bool, + top_k: int = 3, + ) -> str: + example_1 = model_input.examples[0] + example_2 = model_input.examples[1] + example_3 = model_input.examples[2] + prompt = ( + f"{HUMAN_PROMPT}\n" + f"For the following Scenic program, write just the parameter definition section for the input below.{model_input.nat_lang_scene_des}\n" + f"{example_1}{model_input.examples[0]}\n" + f"{example_2}{model_input.examples[1]}\n" + f"{example_3}{model_input.examples[2]}\n" + f"{AI_PROMPT}" + ) + + return prompt + def _few_shot_prompt_with_hyde( self, model_input: ModelInput, @@ -176,6 +258,21 @@ def _few_shot_prompt_with_hyde( return prompt + def _ast_feedback_prompt( + self, + model_input: ModelInput, + verbose: bool, + top_k: int = 3, + ) -> str: + prompt = get_few_shot_ast_prompt(model_input=model_input) + + if verbose: + print(f"Anthropic Model {self._model.value}\n" + f"_few_shot_ast_prompt: {prompt}") + + prompt = cast(str, prompt) + return prompt + def _zero_shot_prompt( self, model_input: ModelInput, @@ -197,7 +294,9 @@ def _zero_shot_prompt( f"_zero_shot_prompt: {prompt}") return prompt - + + + def _format_discussion_prompt( self, model_input: ModelInput, @@ -279,8 +378,14 @@ def _format_message( msg = self._few_shot_prompt_with_hyde(model_input=model_input, verbose=verbose) elif prompt_type == LLMPromptType.PREDICT_TOT_THEN_HYDE: msg = self._few_shot_reasoning_hyde(model_input=model_input, verbose=verbose) + elif prompt_type == LLMPromptType.PREDICT_TOT_THEN_SPLIT: + msg = self._few_shot_reasoning_split(model_input=model_input, verbose=verbose) + elif prompt_type == LLMPromptType.PREDICT_TOT_INTO_NL: + msg = self._few_shot_reasoning_nl(model_input=model_input, verbose=verbose) elif prompt_type == LLMPromptType.EXPERT_SYNTHESIS: msg = self._format_expert_synthesis_prompt(model_input=model_input, verbose=verbose) + elif prompt_type == LLMPromptType.AST_FEEDBACK: + msg = self._ast_feedback_prompt(model_input=model_input, verbose=verbose) else: raise ValueError(f"Invalid prompt type: {prompt_type}") @@ -296,7 +401,9 @@ def _predict( temperature: float, max_length_tokens: int, prompt_type: LLMPromptType, - verbose: bool + verbose: bool, + max_retries: int, + verbose_retries: bool, ) -> str: # to prevent misuse of file handlers limits = httpx.Limits(max_keepalive_connections=1, max_connections=1) @@ -384,6 +491,217 @@ def _predict( max_tokens_to_sample=max_length_tokens, model=self._model.value, ) + elif prompt_type == LLMPromptType.PREDICT_TOT_THEN_SPLIT: + # 1. Use tree of thought to answer all questions in the prompt + panel_answers = [] + for _ in range(NUM_EXPERTS): + claude_response = claude.completions.create( + prompt=self._format_message(model_input=model_input, prompt_type=LLMPromptType.EXPERT_DISCUSSION, verbose=verbose), + temperature=DISCUSSION_TEMPERATURE, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + panel_answers.append(claude_response.completion) + + if len(panel_answers) != NUM_EXPERTS: + raise ValueError(f"Expected {NUM_EXPERTS} panel answers, but got {len(panel_answers)}") + + model_input = ModelInput( + examples=model_input.examples, + nat_lang_scene_des=model_input.nat_lang_scene_des, + first_attempt_scenic_program=model_input.first_attempt_scenic_program, + panel_discussion=panel_answers, + expert_discussion=None # + ) + if verbose: + print(f"Anthropic Model {self._model.value}\n" + f"Tree of Thought: {panel_answers}\n") + + # 2. Ask an expert to synthesize the answers into a single answer + expert_response = claude.completions.create( + prompt=self._format_message(model_input=model_input, prompt_type=LLMPromptType.EXPERT_SYNTHESIS, verbose=verbose), + temperature=DISCUSSION_TEMPERATURE, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + expert_synthesis = expert_response.completion + + if verbose: + print(f"Anthropic Model {self._model.value}\n" + f"Expert Synthesis: {expert_synthesis}\n") + print(f"Anthropic Model {self._model.value}\n" + f"Expert Synthesis: {expert_synthesis}\n") + reasoning = expert_synthesis + print(reasoning) + reasoning_list = reasoning.split(r'\n\d\.') + for item in reasoning_list: + print('>>', item[:20]) + + # self._few_shot_reasoning_split(model_input=model_input, verbose=False, top_k=3, reasoning=expert_synthesis) + incremental_scenic_program = '# Start of scenic program' + + split_prompt = get_discussion_to_split_prompt() + split_prompt = split_prompt.split('\n**\n') + sections = ["## 1. Parameter Definitions:", "## 2. Scene Setup:", "## 3. Behavior Definitions:", "## 4. Assignments:", "## 5. Constraints"] + + # 3. Do a few shot predict on the natural language description + for rest_index, rest_prompt in enumerate(split_prompt[1:]): + reasoning_prompt = "Please concisely output the expert discussion answers most relevant to the requested task of writing part of a Scenic Program. Include NO OTHER input or references to question numbers as we are reading this out loud to a big crowd." + reasoning_prompt = ( + f"{HUMAN_PROMPT}\n" + f"\n** Natural Language Scene Description **\n" + f"{model_input.nat_lang_scene_des}" + f"\n**Expert Discussion**\n" + f"{expert_synthesis}" + f"\n**Task Description**\n" + f"{rest_prompt}" + f"\n**Reasoning Prompt **\n" + f"{reasoning_prompt}" + f"" + f"{AI_PROMPT}" + ) + + reasoning_summary = claude.completions.create( + prompt=reasoning_prompt, + temperature=temperature, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + print(f'<<<<\n{reasoning_summary.completion}\n>>>>') + + head_prompt = split_prompt[0].format( + example_1=model_input.examples[0], + natural_language_description=model_input.nat_lang_scene_des, + reasoning_summary=reasoning_summary.completion, + partial_scenic_program=incremental_scenic_program) + combo_prompt = head_prompt + rest_prompt + + combo_prompt = ( + f"{HUMAN_PROMPT}\n" + f"{combo_prompt}" + f"{AI_PROMPT}" + ) + + claude_response = claude.completions.create( + prompt=combo_prompt, + temperature=temperature, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + incremental_scenic_program += sections[rest_index] + incremental_scenic_program += claude_response.completion + + print('¡¡¡¡') + print(incremental_scenic_program) + print('!!!!') + + stitch_prompt = ( + f"{HUMAN_PROMPT}\n" + # f"Please connect together the following program snippets by deleting any redundant lines of text. You CANNOT make any other changes. The final output will be executed directly so please do not output any leading or trailing text. I will be beheaded if any program segment is missing. Thanks honey.\n" + f"Please delete any lines of code that are redundant between sections. You CANNOT make any other changes. The final output will be executed directly so please do not output any leading or trailing text. I will be beheaded if any program segment is missing. Thanks honey love you.\n" + f"\n{incremental_scenic_program}\n" + f"{AI_PROMPT}" + ) + + claude_response = claude.completions.create( + prompt=stitch_prompt, + temperature=temperature, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + print('((((') + print(claude_response.completion) + print('))))') + # 4. Use the resulting program to query the index to do HyDE thus obtaining the top k programs + # We need to call Claude again + # new_model_input = ModelInput( + # examples=model_input.examples, # this will get overwritten by the search query + # nat_lang_scene_des=model_input.nat_lang_scene_des, + # first_attempt_scenic_program=claude_response.completion, # this is ONLY used for the query search in HyDE/RAG + # expert_discussion=expert_synthesis, + # panel_discussion=panel_answers + # ) + + # 5. Use the top k programs as examples for the few shot prediction along with the answer from the tree of thought + # claude_response = claude.completions.create( + # prompt=self._format_message(model_input=new_model_input, prompt_type=prompt_type, verbose=verbose), + # temperature=temperature, + # max_tokens_to_sample=max_length_tokens, + # model=self._model.value, + # ) + elif prompt_type == LLMPromptType.PREDICT_TOT_INTO_NL: + # 1. Use tree of thought to answer all questions in the prompt + panel_answers = [] + for _ in range(NUM_EXPERTS): + claude_response = claude.completions.create( + prompt=self._format_message(model_input=model_input, prompt_type=LLMPromptType.EXPERT_DISCUSSION, verbose=verbose), + temperature=DISCUSSION_TEMPERATURE, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + panel_answers.append(claude_response.completion) + + if len(panel_answers) != NUM_EXPERTS: + raise ValueError(f"Expected {NUM_EXPERTS} panel answers, but got {len(panel_answers)}") + + model_input = ModelInput( + examples=model_input.examples, + nat_lang_scene_des=model_input.nat_lang_scene_des, + first_attempt_scenic_program=model_input.first_attempt_scenic_program, + panel_discussion=panel_answers, + expert_discussion=None # + ) + if verbose: + print(f"Anthropic Model {self._model.value}\n" + f"Tree of Thought: {panel_answers}\n") + + # 2. Ask an expert to synthesize the answers into a single answer + expert_response = claude.completions.create( + prompt=self._format_message(model_input=model_input, prompt_type=LLMPromptType.EXPERT_SYNTHESIS, verbose=verbose), + temperature=DISCUSSION_TEMPERATURE, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + expert_synthesis = expert_response.completion + + if verbose: + print(f"Anthropic Model {self._model.value}\n" + f"Expert Synthesis: {expert_synthesis}\n") + + # 3. Create a new model input + new_model_input = ModelInput( + examples=model_input.examples, # this will get overwritten by the search query + nat_lang_scene_des=model_input.nat_lang_scene_des, + first_attempt_scenic_program=claude_response.completion, # this is ONLY used for the query search in HyDE/RAG + expert_discussion=expert_synthesis, + panel_discussion=panel_answers + ) + + # 3. Add the expert_synthesis directly into the natural language description + # USING PREDICT_TOT_INTO_NL PROMPT - needs fixes for chattiness + claude_response = claude.completions.create( + prompt=self._format_message(model_input=new_model_input, prompt_type=LLMPromptType.PREDICT_TOT_INTO_NL, verbose=verbose), + temperature=temperature, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + + # 4. Final new model input + final_model_input = ModelInput( + examples=model_input.examples, # this will get overwritten by the search query + nat_lang_scene_des=claude_response.completion, + # first_attempt_scenic_program=claude_response.completion, # this is ONLY used for the query search in HyDE/RAG + # expert_discussion=expert_synthesis, + # panel_discussion=panel_answers + ) + + # 5. Do a few shot predict on the natural language description + claude_response = claude.completions.create( + prompt=self._format_message(model_input=final_model_input, prompt_type=LLMPromptType.PREDICT_FEW_SHOT, verbose=verbose), + temperature=temperature, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) else: claude_response = claude.completions.create( prompt=self._format_message(model_input=model_input, prompt_type=prompt_type, verbose=verbose), @@ -391,6 +709,60 @@ def _predict( max_tokens_to_sample=max_length_tokens, model=self._model.value, ) - - return claude_response.completion + + model_result = str(claude_response.completion) + print('¿¿¿¿') + print(model_result) + print('????') + + with tempfile.TemporaryDirectory(dir=os.curdir) as temp_dir: + retries = max_retries + retries_dir = os.path.join(temp_dir, 'temp_dir') + os.makedirs(retries_dir) + + with tempfile.NamedTemporaryFile(dir=retries_dir, delete=False, suffix='.scenic') as temp_file: + fname = temp_file.name + + while retries: + with open(fname, 'w') as f: + pattern = r'\s+param map' + replacement = r'\nparam map' + model_result = re.sub(pattern, replacement, model_result) + f.write(model_result) + try: + # ast = scenic.syntax.parser.parse_file(fname) + scenario = scenic.scenarioFromFile(fname, mode2D=True) + if verbose_retries: print('No error!') + retries = 0 # If this statement is reached program worked -> terminates loop + except Exception as e: + if verbose_retries: print(f'Retrying... {retries}') + try: + error_message = f"Error details below..\nmessage: {str(e)}\ntext: {e.text}\nlineno: {e.lineno}\nend_lineno: {e.end_lineno}\noffset: {e.offset}\nend_offset: {e.end_offset}" + if verbose_retries: print(error_message) + except: + error_message = f'Error details below..\nmessage: {str(e)}' + if verbose_retries: print(error_message) + + # Constructing correcting claude call + new_model_input = ModelInput( + examples=model_input.examples, # this will get overwritten by the search query + nat_lang_scene_des=model_input.nat_lang_scene_des, + first_attempt_scenic_program=str(model_result), + compiler_error=error_message + ) + + claude_response = claude.completions.create( + prompt=self._format_message(model_input=new_model_input, prompt_type=prompt_type, verbose=verbose), + temperature=temperature, + max_tokens_to_sample=max_length_tokens, + model=self._model.value, + ) + model_result = str(claude_response.completion) + print('°°°°') + print(model_result) + print('^^^^') + retries -= 1 + if verbose_retries: print(model_result) + print(f'[[[[\n{model_result}\n]]]]') + return model_result \ No newline at end of file diff --git a/src/scenicNL/adapters/model_adapter.py b/src/scenicNL/adapters/model_adapter.py index 15ad0c1..34ba633 100644 --- a/src/scenicNL/adapters/model_adapter.py +++ b/src/scenicNL/adapters/model_adapter.py @@ -49,6 +49,8 @@ def _predict( max_length_tokens: int, prompt_type: LLMPromptType, verbose: bool, + max_retries: int, + verbose_retries: bool, ) -> str: """Perform a single prediction. Must be overriden by subclasses.""" raise NotImplementedError @@ -80,7 +82,9 @@ def _batch_processor( cache: Cache, prompt_type: LLMPromptType, ignore_cache: bool, - verbose: bool = False, + verbose: bool, + max_retries: int, + verbose_retries: bool, ) -> Callable[[ModelInput], list[str | APIError]]: """ Return a function that takes a list of model inputs and returns a @@ -114,6 +118,8 @@ def process_single( max_length_tokens=max_tokens, prompt_type=prompt_type, verbose=verbose, + max_retries=max_retries, + verbose_retries=verbose_retries, ) if prompt_type.value == LLMPromptType.PREDICT_PYTHON_API.value: api_input = ModelInput(model_input.examples, prediction) @@ -146,6 +152,8 @@ def predict_batch( verbose: bool = False, num_workers: int = 10, ignore_cache: bool = False, + max_retries: int = 0, + verbose_retries: bool = False, ) -> Iterable[list[str | APIError]]: """ Given a stream of model inputs, return a stream of predictions. This @@ -168,6 +176,8 @@ def predict_batch( prompt_type=prompt_type, ignore_cache=ignore_cache, verbose=verbose, + max_retries=max_retries, + verbose_retries=verbose_retries, ) with ThreadPool(num_workers) as pool: diff --git a/src/scenicNL/adapters/prompts/few_shot_ast.txt b/src/scenicNL/adapters/prompts/few_shot_ast.txt new file mode 100644 index 0000000..a1cd378 --- /dev/null +++ b/src/scenicNL/adapters/prompts/few_shot_ast.txt @@ -0,0 +1,121 @@ + + +Human: +Scenic is a probabilistic programming language for modeling the environments of autonomous cars. A Scenic program defines a distribution over scenes, configurations of physical objects and agents. Scenic can also define (probabilistic) policies for dynamic agents, allowing modeling scenarios where agents take actions over time in response to the state of the world. We use CARLA to render the scenes and simulate the agents. + +Here is one example of a fully compiling Scenic program: +{example_1} + +Modify the Scenic program you wrote that models the description based on: + +1. The following natural language description: +{natural_language_description} + +2. The following scenic program with compiler errors that models the description: +{first_attempt_scenic_program} + +3. The first compiler error raised with the scenic program: +{compiler_error} + +Please output a modified version of your scenic_program modified so the compiler error does not appear. + +OUTPUT NO OTHER LEADING OR TRAILING TEXT OR WHITESPACE BESIDES THE CORRECTED SCENIC PROGRAM. NO ONE CARES. +Output only the natural language description of the scenario as a comment at the top of the file and your proposed Scenic program. DO NOT output any other text or whitespace so I can run your program and see the results. + +Scenic references: + +Here is the list of distributions that are supported in Scenic: +Range(low, high) - Uniform distribution over the range [low, high] +DiscreteRange(low, high) - Uniform distribution over the discreet integer range [low, high] +Normal(mean, std) - Normal distribution with mean and standard deviation +TruncatedNormal(mean, stdDev, low, high) - Normal distribution with mean and standard deviation truncated to the range [low, high] +Uniform(value, …) - Uniform distribution over the values provided +Discrete([value: weight, … ]) - Discrete distribution over the values provided with the given weights + +Here are the only behaviors that are allowed for vehicles. Feel free to select more than one as they are composable: +behavior ConstantThrottleBehavior(x : float): +behavior DriveAvoidingCollisions(target_speed : float = 25, avoidance_threshold : float = 10): + # Drive at a target speed, avoiding collisions with other vehicles + # Throttle is off and braking is applied if the distance to the nearest vehicle is less + # than the avoidance threshold +behavior AccelerateForwardBehavior(): # Accelerate forward with throttle set to 0.5 +behavior FollowLaneBehavior(target_speed : float = 10, laneToFollow : Lane = None, is_oppositeTraffic : bool = False): + # Follow's the lane on which the vehicle is at, unless the laneToFollow is specified. + # Once the vehicle reaches an intersection, by default, the vehicle will take the straight route. + # If straight route is not available, then any available turn route will be taken, uniformly randomly. + # If turning at the intersection, the vehicle will slow down to make the turn, safely. + # This behavior does not terminate. A recommended use of the behavior is to accompany it with condition, + # e.g. do FollowLaneBehavior() until ... + # :param target_speed: Its unit is in m/s. By default, it is set to 10 m/s + # :param laneToFollow: If the lane to follow is different from the lane that the vehicle is on, this parameter can be used to specify that lane. By default, this variable will be set to None, which means that the vehicle will follow the lane that it is currently on. +behavior FollowTrajectoryBehavior(target_speed : float = 10, trajectory : List[Lane] = None, turn_speed : float = None): + # Follows the given trajectory. The behavior terminates once the end of the trajectory is reached. + # :param target_speed: Its unit is in m/s. By default, it is set to 10 m/s + # :param trajectory: It is a list of sequential lanes to track, from the lane that the vehicle is initially on to the lane it should end up on. +behavior TurnBehavior(trajectory : List[Lane] = None, target_speed : float = 6): + # This behavior uses a controller specifically tuned for turning at an intersection. + # This behavior is only operational within an intersection, it will terminate if the vehicle is outside of an intersection. +behavior LaneChangeBehavior(laneSectionToSwitchTo : Lane, is_oppositeTraffic : bool = False, target_speed : float = 10): + # is_oppositeTraffic should be specified as True only if the laneSectionToSwitch to has + # the opposite traffic direction to the initial lane from which the vehicle started LaneChangeBehavior + +Here are the only behaviors that are allowed for pedestrians. Feel free to select more than one as they are composable: +behavior WalkForwardBehavior(speed=0.5): + take SetWalkingDirectionAction(self.heading), SetWalkingSpeedAction(speed) + # Walk forward behavior for pedestrians by uniformly sampling either side of the sidewalk for the pedestrian to walk on +behavior WalkBehavior(maxSpeed=1.4): + take SetWalkAction(True, maxSpeed) +behavior CrossingBehavior(reference_actor, min_speed=1, threshold=10, final_speed=None): + # This behavior dynamically controls the speed of an actor that will perpendicularly (or close to) + # cross the road, so that it arrives at a spot in the road at the same time as a reference actor. + # Args: + # min_speed (float): minimum speed of the crossing actor. As this is a type of "synchronization action", + # a minimum speed is needed, to allow the actor to keep moving even if the reference actor has stopped + # threshold (float): starting distance at which the crossing actor starts moving + # final_speed (float): speed of the crossing actor after the reference one surpasses it + +Here is the full set of vehicles supported in Scenic. +"Audi - A2": "vehicle.audi.a2", +"Audi - E-Tron": "vehicle.audi.etron", +"Audi - TT": "vehicle.audi.tt", +"BMW - Gran Tourer": "vehicle.bmw.grandtourer", +"Chevrolet - Impala": "vehicle.chevrolet.impala", +"Citroen - C3": "vehicle.citroen.c3", +"Dodge - Charger 2020": "vehicle.dodge.charger_2020", +"Dodge - Police Charger": "vehicle.dodge.charger_police", +"Dodge - Police Charger 2020": "vehicle.dodge.charger_police_2020", +"Ford - Crown (taxi)": "vehicle.ford.crown", +"Ford - Mustang": "vehicle.ford.mustang", +"Jeep - Wrangler Rubicon": "vehicle.jeep.wrangler_rubicon", +"Lincoln - MKZ 2017": "vehicle.lincoln.mkz_2017", +"Lincoln - MKZ 2020": "vehicle.lincoln.mkz_2020", +"Mercedes - Coupe": "vehicle.mercedes.coupe", +"Mercedes - Coupe 2020": "vehicle.mercedes.coupe_2020", +"Micro - Microlino": "vehicle.micro.microlino", +"Mini - Cooper S": "vehicle.mini.cooper_s", +"Mini - Cooper S 2021": "vehicle.mini.cooper_s_2021", +"Nissan - Micra": "vehicle.nissan.micra", +"Nissan - Patrol": "vehicle.nissan.patrol", +"Nissan - Patrol 2021": "vehicle.nissan.patrol_2021", +"Seat - Leon": "vehicle.seat.leon", +"Tesla - Model 3": "vehicle.tesla.model3", +"Toyota - Prius": "vehicle.toyota.prius", +"CARLA Motors - CarlaCola": "vehicle.carlamotors.carlacola", +"CARLA Motors - European HGV (cab-over-engine type)": "vehicle.carlamotors.european_hgv", +"CARLA Motors - Firetruck": "vehicle.carlamotors.firetruck", +"Tesla - Cybertruck": "vehicle.tesla.cybertruck", +"Ford - Ambulance": "vehicle.ford.ambulance", +"Mercedes - Sprinter": "vehicle.mercedes.sprinter", +"Volkswagen - T2": "vehicle.volkswagen.t2", +"Volkswagen - T2 2021": "vehicle.volkswagen.t2_2021", +"Mitsubishi - Fusorosa": "vehicle.mitsubishi.fusorosa", +"Harley Davidson - Low Rider": "vehicle.harley-davidson.low_rider", +"Kawasaki - Ninja": "vehicle.kawasaki.ninja", +"Vespa - ZX 125": "vehicle.vespa.zx125", +"Yamaha - YZF": "vehicle.yamaha.yzf", +"BH - Crossbike": "vehicle.bh.crossbike", +"Diamondback - Century": "vehicle.diamondback.century", +"Gazelle - Omafiets": "vehicle.gazelle.omafiets" + + +Assistant: \ No newline at end of file diff --git a/src/scenicNL/adapters/prompts/scenic_tutorial_prompt.txt b/src/scenicNL/adapters/prompts/scenic_tutorial_prompt.txt index 136e998..1557b0e 100644 --- a/src/scenicNL/adapters/prompts/scenic_tutorial_prompt.txt +++ b/src/scenicNL/adapters/prompts/scenic_tutorial_prompt.txt @@ -2,7 +2,7 @@ Here is a quick tutorial about the Scenic language. Scenic scripts are typically divided into three sections: parameter definitions, scene setup, and behaviors. 1. Parameter Definitions: - In the parameter definitions section, you handle imports and define any parameters your scenario will use. +In the parameter definitions section, you handle imports and define any parameters your scenario will use. A Scenic script begins with importing necessary libraries. The first lines could be: "param map = localPath('../../../assets/maps/CARLA/Town05.xodr') param carla_map = 'Town05' diff --git a/src/scenicNL/adapters/prompts/tot_nl.txt b/src/scenicNL/adapters/prompts/tot_nl.txt new file mode 100644 index 0000000..9dc47cf --- /dev/null +++ b/src/scenicNL/adapters/prompts/tot_nl.txt @@ -0,0 +1,12 @@ +Scenic is a probabilistic programming language for modeling the environments of autonomous cars. A Scenic program defines a distribution over scenes, configurations of physical objects and agents. Scenic can also define (probabilistic) policies for dynamic agents, allowing modeling scenarios where agents take actions over time in response to the state of the world. We use CARLA to render the scenes and simulate the agents. + +Let's start working towards having you help design a Scenic program by modifying the following natural language description: +{natural_language_description} + +Using the expert and panel reasoning below, please update the previous natural language description to include all the key information and Scenic constructs from the natural language description. + +Expert and panel discussions attached below. +{expert_discussion} + +{panel_discussion} + diff --git a/src/scenicNL/adapters/prompts/tot_split.txt b/src/scenicNL/adapters/prompts/tot_split.txt new file mode 100644 index 0000000..dafe377 --- /dev/null +++ b/src/scenicNL/adapters/prompts/tot_split.txt @@ -0,0 +1,145 @@ +Scenic is a probabilistic programming language for modeling the environments of autonomous cars. A Scenic program defines a distribution over scenes, configurations of physical objects and agents. Scenic can also define (probabilistic) policies for dynamic agents, allowing modeling scenarios where agents take actions over time in response to the state of the world. We use CARLA to render the scenes and simulate the agents. + +** Here is one example of a Scenic program ** +{example_1} + +** Let's start working towards having you create your own Scenic program based on the following natural language description ** +{natural_language_description} + +** From this natural language description a collection of excerpts have compiled the following reasoning information ** +{reasoning_summary} + +** TASK ** +Continue writing the partial Scenic program below. +Given the partial Scenic program and the reasoning excerpt, please continue and write only the next section of the Scenic program. I will APPEND your output directly to my code.scenic file so do not add any extra text or whitespace. + +## PARTIAL SCENIC PROGRAM ## +{partial_scenic_program} +# continue writing from here + +** + +1. Parameter Definitions: +In the parameter definitions section, you handle imports and define any parameters your scenario will use. +A Scenic script begins with importing necessary libraries. +The first lines could be: "param map = localPath('../../../assets/maps/CARLA/Town05.xodr') +param carla_map = 'Town05' +model scenic.simulators.carla.model" to import the simulator library. +The set of maps allowed is only of the format ../../../assets/maps/CARLA/Town01.xodr for Town01, Town02, Town03, Town04, Town05, Town06, Town07. +No other map can be accessed. Please do not access exact streets or lanes as a result. +Then define any scene parameters, for example: "speed = Range(15, 25)" defines a parameter speed with values ranging from 15 to 25. + +Start this section with ## PARAMETER DEFINITIONS + +** + +2. Scene Setup: +In the scene setup section, you describe the static aspects of the scenario. +For example, "lane = Uniform(*network.lanes) +spawnPt = new OrientedPoint on lane.centerline +obstacle = new Trash at spawnPt offset by Range(1, -1) @ 0" creates a Trash obstacle offset from the centerline of a random lane. + +Start this section with ## SCENE SETUP + +** + +3. Behavior Definitions: +In the behavior definition section, you define the dynamic behaviors involved in the scenario. +Objects which can take actions over time are called agents. We specifiy their dynamic behavior using the built-in property, `behavior`. A behavior defines a sequence of actions for the agent to take, which need not be fixed but can be probabilistic and depend on the state of the agent or other objects. In Scenic, an action is an instantaneous operation executed by an agent, like setting the steering angle of a car or turning on its headlights. +To define a behavior, we write a function which runs over the course of the scenario, periodically issuing actions. Scenic uses a discrete notion of time, so at each time step the function specifies zero or more actions for the agent to take. The function can also access the current state of the agent and other objects in the scene, and use this information to determine which actions to take. + +Here is an example of a behavior where a car is waiting for an ego car to be a certain distance away before it starts moving: +behavior WaitUntilClose(threshold=15): + while (distance from self to ego) > threshold: + wait + do FollowLaneBehavior() +Here, we repeatedly query the distance from the agent running the behavior (self) to the ego car; as long as it is above a threshold, we wait, which means take no actions. Once the threshold is met, we start driving by invoking the built in, FollowLaneBehavior. Since FollowLaneBehavior runs forever, we will never return to the WaitUntilClose behavior. + +Unlike ordinary Scenic code, control flow constructs such as if and while are allowed to depend on random variables inside a behavior. Any distributions defined inside a behavior are sampled at simulation time, not during scene sampling. Consider the following behavior: +behavior Foo(): + threshold = Range(4, 7) + while True: + if self.distanceToClosest(Pedestrian) < threshold: + strength = TruncatedNormal(0.8, 0.02, 0.5, 1) + take SetBrakeAction(strength), SetThrottleAction(0) + else: + take SetThrottleAction(0.5), SetBrakeAction(0) +Here, the value of threshold is sampled only once, at the beginning of the scenario when the behavior starts running. The value strength, on the other hand, is sampled every time step when the car is braking we use a slightly different braking strength (0.8 on average, but with 0 mean Gaussian noise added with standard deviation 0.02, truncated to the range [0.5, 1]). The behavior also uses the built-in function distanceToClosest, which returns the distance to the closest object of a given type (here, Pedestrian). This function is evaluated at each time step, so the car will always brake if a pedestrian is within the threshold distance. + +Suppose we want a car that follows a lane, stopping whenever it encounters an obstacle. Scenic provides a concept of interrupts. +behavior FollowAvoidingObstacles(): + try: + do FollowLaneBehavior() + interrupt when self.distanceToClosest(Object) < 5: + take SetBrakeAction(1) +This try-interrupt statement has similar syntax to the Python try statement (and in fact allows except clauses just as in Python), and begins in the same way: at first, the code block after the try: (the body) is executed. At the start of every time step during its execution, the condition from each interrupt clause is checked; if any are true, execution of the body is suspended and we instead begin to execute the corresponding interrupt handler. In the example above, there is only one interrupt, which fires when we come within 5 meters of any object. When that happens, FollowLaneBehavior is paused and we instead apply full braking for one time step. In the next step, we will resume FollowLaneBehavior wherever it left off, unless we are still within 5 meters of an object, in which case the interrupt will fire again. +If there are multiple interrupt clauses, successive clauses take precedence over those which precede them. Furthermore, such higher-priority interrupts can fire even during the execution of an earlier interrupt handler. This makes it easy to model a hierarchy of behaviors with different priorities; for example, we could implement a car which drives along a lane, passing slow cars and avoiding collisions, along the following lines: +behavior Drive(): + try: + do FollowLaneBehavior() + interrupt when self.distanceToNextObstacle() < 20: + do PassingBehavior() + interrupt when self.timeToCollision() < 5: + do CollisionAvoidance() +Alternatively, we can use `until` to specify a condition that must be met before the behavior can complete. +behavior ApproachAndTurnLeft(): + do FollowLaneBehavior() until (distance from self to intersection) < 10 + do WaitForTrafficLightBehavior() + do TurnLeftBehavior() +Or we can do it by time: +behavior DriveForAWhile(): + do FollowLaneBehavior() for 30 seconds +The alternative form `do behavior for n steps` uses time steps instead of real simulation time. + +Start this section with ## BEHAVIORS + +** + +4. Assignments: +In the assignments section, you assign the behaviors to initialized vehicle, bicyclist, and pedestrian objects now that behaviors have been defined. + +If needed, you can use syntax such as "A can see B" to make sure agent B is visible to agent A. +You can also use syntax like "distance from A to B" to make sure the distance between A and B is within a certain range. +oncomingCar = new Car on leftLaneSec.centerline, + with behavior OncomingCarBehavior() + +ego = new Car at spawnPt, + with behavior EgoBehavior(leftLaneSec) + +After this, "ego = new Car following roadDirection from spawnPt for Range(-50, -30), + with blueprint EGO_MODEL, + with behavior EgoBehavior(EGO_SPEED)" defines a dynamic agent with this behavior and other properties. All scenes must have an ego vehicle. + +blockingCar = new Car following roadDirection from ego for BLOCKING_CAR_DIST, + with viewAngle 90 deg + +Start this section with ## ASSIGNMENTS + +** + +5. Constraints: +In the constraints section, you define require statements to enforce initial and termination conditions. +Just as you can declare spatial constraints on scenes using the require statement, you can also impose constraints on dynamic scenarios. For example, if we don’t want to generate any simulations where car1 and car2 are simultaneously visible from the ego car, we could write: +require always not ((ego can see car1) and (ego can see car2)) +Here, `always condition` is a linear temporal (LTL) operator which can only be used inside a requirement, and which evaluates to true if and only if the condition is true at every time step of the scenario. So if the condition above is ever false during a simulation, the requirement will be violated, causing Scenic to reject that simulation and sample a new one. Similarly, we can require that a condition hold at some time during the scenario using the eventually operator: +require eventually ego in intersection + +So, if you need to include to include temporal aspects in your behavior, you use a require statement like so to model that no pedestrian comes close to self until the ego does (after which we place no further restrictions) +behavior WaitUntilClose(threshold=15): + while distance from self to ego > threshold: + require self.distanceToClosest(Pedestrian) > threshold + wait + do FollowLaneBehavior() + +Temporal operators can be combined with Boolean operators in a manner similar to LTL to build up more complex requirements: +require (always car.speed < 30) implies (always distance to car > 10) +Or relational: +require car2 not in intersection until car1 in intersection +require eventually car2 in intersection + +Require and terminate statements can be used to enforce post-conditions that determine that certain events occured or terminate the program after certain conditions are met thus satisfying the requirements of the scenario. Any require or terminate statement should not be about time and should be about events in the scene. For example, to make sure an oncoming Car is at a visible section of the lane: +require blockingCar can see oncomingCar +require (distance from blockingCar to oncomingCar) < DIST_BTW_BLOCKING_ONCOMING_CARS +require (distance from blockingCar to intersection) > DIST_TO_INTERSECTION + +Start this section with ## CONSTRAINTS \ No newline at end of file diff --git a/src/scenicNL/common.py b/src/scenicNL/common.py index 5dbd28e..9e9807b 100644 --- a/src/scenicNL/common.py +++ b/src/scenicNL/common.py @@ -25,9 +25,11 @@ class LLMPromptType(Enum): PREDICT_FEW_SHOT_WITH_HYDE = "predict_few_shot_hyde" PREDICT_FEW_SHOT_WITH_HYDE_TOT = "predict_few_shot_hyde_tot" PREDICT_TOT_THEN_HYDE = "predict_tot_then_hyde" + PREDICT_TOT_THEN_SPLIT = "predict_tot_then_split" + PREDICT_TOT_INTO_NL = "predict_tot_into_nl" EXPERT_DISCUSSION = "expert_discussion" EXPERT_SYNTHESIS = "expert_synthesis" - + AST_FEEDBACK = "ast_feedback" class PromptFiles(Enum): PROMPT_PATH = os.path.join(os.curdir, 'src', 'scenicNL', 'adapters', 'prompts') @@ -38,7 +40,9 @@ class PromptFiles(Enum): SCENIC_TUTORIAL = os.path.join(PROMPT_PATH, 'scenic_tutorial_prompt.txt') TOT_EXPERT_DISCUSSION = os.path.join(PROMPT_PATH, 'tot_questions.txt') EXPERT_SYNTHESIS = os.path.join(PROMPT_PATH, 'expert_synthesis.txt') - + AST_FEEDBACK_CLAUDE = os.path.join(PROMPT_PATH, 'few_shot_ast.txt') + TOT_SPLIT = os.path.join(PROMPT_PATH, 'tot_split.txt') + TOT_NL = os.path.join(PROMPT_PATH, 'tot_nl.txt') @dataclass(frozen=True) class ModelInput: @@ -50,6 +54,7 @@ class ModelInput: examples: list[str] nat_lang_scene_des: str first_attempt_scenic_program: Optional[str] = None + compiler_error: Optional[str] = None expert_discussion: Optional[str] = None panel_discussion: Optional[List[str]] = None @@ -160,6 +165,39 @@ def get_discussion_to_program_prompt() -> str: return prompt +def get_discussion_to_split_prompt() -> str: + prompt = "" + with open(PromptFiles.TOT_SPLIT.value) as f: + prompt = f.read() + return prompt + +def get_few_shot_ast_prompt(model_input) -> str: + prompt = "" + with open(PromptFiles.AST_FEEDBACK_CLAUDE.value) as f: + prompt = f.read() + + prompt = prompt.format( + natural_language_description=model_input.nat_lang_scene_des, + example_1=model_input.examples[0], + example_2=model_input.examples[1], + example_3=model_input.examples[2], + expert_discussion=model_input.expert_discussion, + first_attempt_scenic_program=model_input.first_attempt_scenic_program, + compiler_error=model_input.compiler_error + ) + + return prompt + +def get_tot_nl_prompt(model_input) -> str: + prompt = "" + with open(PromptFiles.TOT_NL.value) as f: + prompt = f.read() + prompt = prompt.format( + natural_language_description=model_input.nat_lang_scene_des, + expert_discussion=model_input.expert_discussion, + panel_discussion=model_input.panel_discussion + ) + return prompt class VectorDB(): def __init__( diff --git a/src/scenicNL/main.py b/src/scenicNL/main.py index ebabd3c..b8ca2d0 100644 --- a/src/scenicNL/main.py +++ b/src/scenicNL/main.py @@ -161,6 +161,20 @@ def main(): help="Number of workers to use for parallel processing.", ) +@click.option( + "--max_retries", + type=click.INT, + default=0, + show_default=True, + help="Maximum number of compiler-in-the-loop retries.", +) + +@click.option( + "--verbose_retries", + is_flag=True, + help="Boolean condition to display or omit verbose retries." +) + def main( query_path: Path, output_path: Path, @@ -178,6 +192,8 @@ def main( should_cache_retry_errors: bool, keep_filename: bool, temperature: float, + max_retries: int, + verbose_retries: bool, ) -> None: """ Generate simulator scenes from natural language descriptions. @@ -248,7 +264,7 @@ def main( scenic_metric_path = os.path.join(scenic_metadata_path, 'metrics.txt') start_time = time.time() - compile_pass, compile_fail, api_error = 0, 0, 0 + compile_pass = compile_fail = execute_pass = execute_fail = api_error = 0 for index, outputs in enumerate(adapter.predict_batch( model_inputs=model_input_list, cache_path=cache_path, @@ -260,6 +276,8 @@ def main( verbose=verbose, num_workers=num_workers, ignore_cache=ignore_cache, + max_retries=max_retries, + verbose_retries=verbose_retries, ) ): for attempt, output in enumerate(outputs): @@ -280,17 +298,30 @@ def main( with open(fname, 'w') as f: f.write(output) try: - scenic.scenarioFromFile(fname, mode2D=True) - fname_compile = os.path.join(result_path, f'{fstub}.scenic') - with open(fname_compile, 'w') as f: - f.write(output) - print(f'No errors when compiling input {debug}') + ast = scenic.syntax.parser.parse_file(fname) + print(f'Compiled input {debug} successfully: {ast}') + # print(f'No errors when compiling input {debug}') compile_pass += 1 except Exception as e: print(f'Error while compiling for input {debug}: {e}') compile_fail += 1 with open(scenic_error_path, 'a') as f: - f.write(f'{index} - {fstub}: {e}\n') + f.write(f'{index} - {fstub} compile error: {e}\n') + try: + scenario = scenic.scenarioFromFile(fname, mode2D=True) + print(f'Executed input {debug} successfully: {scenario}') + + fsave = os.path.join(result_path, f'{fstub}.scenic') + execute_pass += 1 + with open(fsave, 'w') as f: + f.write(output) + # print(f'No errors when compiling input {debug}') + except Exception as e: + print(f'Error while executing for input {debug}: {e}') + execute_fail += 1 + with open(scenic_error_path, 'a') as f: + f.write(f'{index} - {fstub} execute error: {e}\n') + print(f'Output path {debug} at: {fname}') print('----------------\n\n') end_time = time.time() @@ -298,13 +329,16 @@ def main( api_error_rate = round((100*api_error/total), 2) compile_rate = round((100*compile_pass/total), 2) + execute_rate = round((100*execute_pass/total), 2) eval_rate = round((end_time-start_time)/total, 5) - print(f'API error rate: {api_error_rate}%') - print(f'Compilation success rate: {compile_rate}%') + print(f'Compile success rate: {compile_rate}%') + print(f'Execute success rate: {execute_rate}%') + print(f'## API error rate ##: {api_error_rate}%') print(scenic_metric_path) with open(scenic_metric_path, 'w') as f: f.write(f'Compile rate: {compile_rate}\n') + f.write(f'Execute rate: {execute_rate}\n') f.write(f'API error rate: {api_error_rate}\n') f.write(f'Secs per program: {eval_rate}\n')